SherbyNet - /page/wikibook/entry.mu

Micron Document

<!DOCTYPE html>

<title>AI alignment</title>

</head>

AI alignment

</h1>

</header>

/* start https://en.wikipedia.org/ */

.mw-parser-output .hlist dl,.mw-parser-output .hlist ol,.mw-parser-output .hlist ul{margin:0;padding:0}.mw-parser-output .hlist dd,.mw-parser-output .hlist dt,.mw-parser-output .hlist li{margin:0;display:inline}.mw-parser-output .hlist.inline,.mw-parser-output .hlist.inline dl,.mw-parser-output .hlist.inline ol,.mw-parser-output .hlist.inline ul,.mw-parser-output .hlist dl dl,.mw-parser-output .hlist dl ol,.mw-parser-output .hlist dl ul,.mw-parser-output .hlist ol dl,.mw-parser-output .hlist ol ol,.mw-parser-output .hlist ol ul,.mw-parser-output .hlist ul dl,.mw-parser-output .hlist ul ol,.mw-parser-output .hlist ul ul{display:inline}.mw-parser-output .hlist .mw-empty-li{display:none}.mw-parser-output .hlist dt::after{content:": "}.mw-parser-output .hlist dd::after,.mw-parser-output .hlist li::after{content:" · ";font-weight:bold}.mw-parser-output .hlist dd:last-child::after,.mw-parser-output .hlist dt:last-child::after,.mw-parser-output .hlist li:last-child::after{content:none}.mw-parser-output .hlist dd dd:first-child::before,.mw-parser-output .hlist dd dt:first-child::before,.mw-parser-output .hlist dd li:first-child::before,.mw-parser-output .hlist dt dd:first-child::before,.mw-parser-output .hlist dt dt:first-child::before,.mw-parser-output .hlist dt li:first-child::before,.mw-parser-output .hlist li dd:first-child::before,.mw-parser-output .hlist li dt:first-child::before,.mw-parser-output .hlist li li:first-child::before{content:" (";font-weight:normal}.mw-parser-output .hlist dd dd:last-child::after,.mw-parser-output .hlist dd dt:last-child::after,.mw-parser-output .hlist dd li:last-child::after,.mw-parser-output .hlist dt dd:last-child::after,.mw-parser-output .hlist dt dt:last-child::after,.mw-parser-output .hlist dt li:last-child::after,.mw-parser-output .hlist li dd:last-child::after,.mw-parser-output .hlist li dt:last-child::after,.mw-parser-output .hlist li li:last-child::after{content:")";font-weight:normal}.mw-parser-output .hlist ol{counter-reset:listitem}.mw-parser-output .hlist ol>li{counter-increment:listitem}.mw-parser-output .hlist ol>li::before{content:" "counter(listitem)"\a0 "}.mw-parser-output .hlist dd ol>li:first-child::before,.mw-parser-output .hlist dt ol>li:first-child::before,.mw-parser-output .hlist li ol>li:first-child::before{content:" ("counter(listitem)"\a0 "}

/* end https://en.wikipedia.org/ */

</style><style data-mw-deduplicate="TemplateStyles:r1246091330">

/* start https://en.wikipedia.org/ */

.mw-parser-output .sidebar{width:22em;float:right;clear:right;margin:0.5em 0 1em 1em;background:var(--background-color-neutral-subtle,#f8f9fa);border:1px solid var(--border-color-base,#a2a9b1);padding:0.2em;text-align:center;line-height:1.4em;font-size:88%;border-collapse:collapse;display:table}body.skin-minerva .mw-parser-output .sidebar{display:table!important;float:right!important;margin:0.5em 0 1em 1em!important}.mw-parser-output .sidebar-subgroup{width:100%;margin:0;border-spacing:0}.mw-parser-output .sidebar-left{float:left;clear:left;margin:0.5em 1em 1em 0}.mw-parser-output .sidebar-none{float:none;clear:both;margin:0.5em 1em 1em 0}.mw-parser-output .sidebar-outer-title{padding:0 0.4em 0.2em;font-size:125%;line-height:1.2em;font-weight:bold}.mw-parser-output .sidebar-top-image{padding:0.4em}.mw-parser-output .sidebar-top-caption,.mw-parser-output .sidebar-pretitle-with-top-image,.mw-parser-output .sidebar-caption{padding:0.2em 0.4em 0;line-height:1.2em}.mw-parser-output .sidebar-pretitle{padding:0.4em 0.4em 0;line-height:1.2em}.mw-parser-output .sidebar-title,.mw-parser-output .sidebar-title-with-pretitle{padding:0.2em 0.8em;font-size:145%;line-height:1.2em}.mw-parser-output .sidebar-title-with-pretitle{padding:0.1em 0.4em}.mw-parser-output .sidebar-image{padding:0.2em 0.4em 0.4em}.mw-parser-output .sidebar-heading{padding:0.1em 0.4em}.mw-parser-output .sidebar-content{padding:0 0.5em 0.4em}.mw-parser-output .sidebar-content-with-subgroup{padding:0.1em 0.4em 0.2em}.mw-parser-output .sidebar-above,.mw-parser-output .sidebar-below{padding:0.3em 0.8em;font-weight:bold}.mw-parser-output .sidebar-collapse .sidebar-above,.mw-parser-output .sidebar-collapse .sidebar-below{border-top:1px solid #aaa;border-bottom:1px solid #aaa}.mw-parser-output .sidebar-navbar{text-align:right;font-size:115%;padding:0 0.4em 0.4em}.mw-parser-output .sidebar-list-title{padding:0 0.4em;text-align:left;font-weight:bold;line-height:1.6em;font-size:105%}.mw-parser-output .sidebar-list-title-c{padding:0 0.4em;text-align:center;margin:0 3.3em}@media(max-width:640px){body.mediawiki .mw-parser-output .sidebar{width:100%!important;clear:both;float:none!important;margin-left:0!important;margin-right:0!important}}body.skin--responsive .mw-parser-output .sidebar a>img{max-width:none!important}@media screen{html.skin-theme-clientpref-night .mw-parser-output .sidebar:not(.notheme) .sidebar-list-title,html.skin-theme-clientpref-night .mw-parser-output .sidebar:not(.notheme) .sidebar-title-with-pretitle{background:transparent!important}html.skin-theme-clientpref-night .mw-parser-output .sidebar:not(.notheme) .sidebar-title-with-pretitle a{color:var(--color-progressive)!important}}@media screen and (prefers-color-scheme:dark){html.skin-theme-clientpref-os .mw-parser-output .sidebar:not(.notheme) .sidebar-list-title,html.skin-theme-clientpref-os .mw-parser-output .sidebar:not(.notheme) .sidebar-title-with-pretitle{background:transparent!important}html.skin-theme-clientpref-os .mw-parser-output .sidebar:not(.notheme) .sidebar-title-with-pretitle a{color:var(--color-progressive)!important}}@media print{body.ns-0 .mw-parser-output .sidebar{display:none!important}}

/* end https://en.wikipedia.org/ */

</style><table class="sidebar sidebar-collapse nomobile nowraplinks hlist"><tbody><tr><td class="sidebar-pretitle">Part of a series on</td></tr><tr><th class="sidebar-title-with-pretitle"><a href="Artificial_intelligence" title="Artificial intelligence">Artificial intelligence (AI)</a></th></tr><tr><td class="sidebar-image"></td></tr><tr><td class="sidebar-content">

<div class="sidebar-list mw-collapsible mw-collapsed"><div class="sidebar-list-title" style="text-align:center;color: var(--color-base)"><a href="Artificial_intelligence#Goals" title="Artificial intelligence">Major goals</a></div><div class="sidebar-list-content mw-collapsible-content">

<ul><li><a href="Artificial_general_intelligence" title="Artificial general intelligence">Artificial general intelligence</a></li>

<li><a href="Intelligent_agent" title="Intelligent agent">Intelligent agent</a></li>

<li><a href="Recursive_self-improvement" title="Recursive self-improvement">Recursive self-improvement</a></li>

<li><a href="Automated_planning_and_scheduling" title="Automated planning and scheduling">Planning</a></li>

<li><a href="Computer_vision" title="Computer vision">Computer vision</a></li>

<li><a href="General_game_playing" title="General game playing">General game playing</a></li>

<li><a href="Knowledge_representation_and_reasoning" title="Knowledge representation and reasoning">Knowledge representation</a></li>

<li><a href="Natural_language_processing" title="Natural language processing">Natural language processing</a></li>

<li><a href="Robotics" title="Robotics">Robotics</a></li>

<li><a href="AI_safety" title="AI safety">AI safety</a></li></ul></div></div></td>

</tr><tr><td class="sidebar-content">

<div class="sidebar-list mw-collapsible mw-collapsed"><div class="sidebar-list-title" style="text-align:center;color: var(--color-base)">Approaches</div><div class="sidebar-list-content mw-collapsible-content">

<ul><li><a href="Machine_learning" title="Machine learning">Machine learning</a></li>

<li><a href="Symbolic_artificial_intelligence" title="Symbolic artificial intelligence">Symbolic</a></li>

<li><a href="Deep_learning" title="Deep learning">Deep learning</a></li>

<li><a href="Bayesian_network" title="Bayesian network">Bayesian networks</a></li>

<li><a href="Evolutionary_algorithm" title="Evolutionary algorithm">Evolutionary algorithms</a></li>

<li><a href="Hybrid_intelligent_system" title="Hybrid intelligent system">Hybrid intelligent systems</a></li>

<li><a href="Artificial_intelligence_systems_integration" title="Artificial intelligence systems integration">Systems integration</a></li></ul></div></div></td>

</tr><tr><td class="sidebar-content">

<div class="sidebar-list mw-collapsible mw-collapsed"><div class="sidebar-list-title" style="text-align:center;color: var(--color-base)"><a href="Applications_of_artificial_intelligence" title="Applications of artificial intelligence">Applications</a></div><div class="sidebar-list-content mw-collapsible-content">

<ul><li><a href="Machine_learning_in_bioinformatics" title="Machine learning in bioinformatics">Bioinformatics</a></li>

<li><a href="Deepfake" title="Deepfake">Deepfake</a></li>

<li><a href="Machine_learning_in_earth_sciences" title="Machine learning in earth sciences">Earth sciences</a></li>

<li><a href="Applications_of_artificial_intelligence#Finance" title="Applications of artificial intelligence"> Finance </a></li>

<li><a href="Generative_artificial_intelligence" title="Generative artificial intelligence">Generative AI</a>

<li><a href="Generative_audio" title="Generative audio">Audio</a></li>

<li><a href="Music_and_artificial_intelligence" title="Music and artificial intelligence">Music</a></li></ul></li>

<li><a href="Artificial_intelligence_in_government" title="Artificial intelligence in government">Government</a></li>

<li><a href="Artificial_intelligence_in_healthcare" title="Artificial intelligence in healthcare">Healthcare</a>

<ul><li><a href="Artificial_intelligence_in_mental_health" title="Artificial intelligence in mental health">Mental health</a></li></ul></li>

<li><a href="Artificial_intelligence_in_industry" title="Artificial intelligence in industry">Industry</a></li>

<li><a href="AI-assisted_software_development" title="AI-assisted software development">Software development</a></li>

<li><a href="Machine_translation" title="Machine translation">Translation</a></li>

<li><a href="Artificial_intelligence_arms_race" title="Artificial intelligence arms race"> Military </a></li>

<li><a href="Machine_learning_in_physics" title="Machine learning in physics">Physics</a></li>

<li><a href="List_of_artificial_intelligence_projects" title="List of artificial intelligence projects">Projects</a></li></ul></div></div></td>

</tr><tr><td class="sidebar-content">

<div class="sidebar-list mw-collapsible mw-collapsed"><div class="sidebar-list-title" style="text-align:center;color: var(--color-base)"><a href="Philosophy_of_artificial_intelligence" title="Philosophy of artificial intelligence">Philosophy</a></div><div class="sidebar-list-content mw-collapsible-content">

<ul><li><a href="Artificial_consciousness" title="Artificial consciousness">Artificial consciousness</a></li>

<li><a href="Chinese_room" title="Chinese room">Chinese room</a></li>

<li><a href="Friendly_artificial_intelligence" title="Friendly artificial intelligence">Friendly AI</a></li>

<li><a href="AI_control_problem" class="mw-redirect" title="AI control problem">Control problem</a>/<a href="AI_takeover" title="AI takeover">Takeover</a></li>

<li><a href="Ethics_of_artificial_intelligence" title="Ethics of artificial intelligence">Ethics</a></li>

<li><a href="Existential_risk_from_artificial_general_intelligence" class="mw-redirect" title="Existential risk from artificial general intelligence">Existential risk</a></li>

<li><a href="Turing_test" title="Turing test">Turing test</a></li>

<li><a href="Uncanny_valley" title="Uncanny valley">Uncanny valley</a></li></ul></div></div></td>

</tr><tr><td class="sidebar-content">

<div class="sidebar-list mw-collapsible mw-collapsed"><div class="sidebar-list-title" style="text-align:center;color: var(--color-base)"><a href="History_of_artificial_intelligence" title="History of artificial intelligence">History</a></div><div class="sidebar-list-content mw-collapsible-content">

<ul><li><a href="Timeline_of_artificial_intelligence" title="Timeline of artificial intelligence">Timeline</a></li>

<li><a href="Progress_in_artificial_intelligence" title="Progress in artificial intelligence">Progress</a></li>

<li><a href="AI_winter" title="AI winter">AI winter</a></li>

</tr><tr><td class="sidebar-content">

<div class="sidebar-list mw-collapsible mw-collapsed"><div class="sidebar-list-title" style="text-align:center;color: var(--color-base)">Glossary</div><div class="sidebar-list-content mw-collapsible-content">

<ul><li><a href="Glossary_of_artificial_intelligence" title="Glossary of artificial intelligence">Glossary</a></li></ul></div></div></td>

</tr><tr><td class="sidebar-navbar"><style data-mw-deduplicate="TemplateStyles:r1239400231">

/* start https://en.wikipedia.org/ */

.mw-parser-output .navbar{display:inline;font-size:88%;font-weight:normal}.mw-parser-output .navbar-collapse{float:left;text-align:left}.mw-parser-output .navbar-boxtext{word-spacing:0}.mw-parser-output .navbar ul{display:inline-block;white-space:nowrap;line-height:inherit}.mw-parser-output .navbar-brackets::before{margin-right:-0.125em;content:"[ "}.mw-parser-output .navbar-brackets::after{margin-left:-0.125em;content:" ]"}.mw-parser-output .navbar li{word-spacing:-0.125em}.mw-parser-output .navbar a>span,.mw-parser-output .navbar a>abbr{text-decoration:inherit}.mw-parser-output .navbar-mini abbr{font-variant:small-caps;border-bottom:none;text-decoration:none;cursor:inherit}.mw-parser-output .navbar-ct-full{font-size:114%;margin:0 7em}.mw-parser-output .navbar-ct-mini{font-size:114%;margin:0 4em}html.skin-theme-clientpref-night .mw-parser-output .navbar li a abbr{color:var(--color-base)!important}@media(prefers-color-scheme:dark){html.skin-theme-clientpref-os .mw-parser-output .navbar li a abbr{color:var(--color-base)!important}}@media print{.mw-parser-output .navbar{display:none!important}}

/* end https://en.wikipedia.org/ */

</style></td></tr></tbody></table>

In the field of <a href="Artificial_intelligence" title="Artificial intelligence">artificial intelligence</a> (AI), alignment aims to steer AI systems toward a person's or group's intended goals, preferences, or ethical principles. An AI system is considered aligned if it advances the intended objectives. A misaligned AI system pursues unintended objectives.<a href="#cite_note-aima4-1">[1]</a>

It is often challenging for AI designers to align an AI system because it is difficult for them to specify the full range of desired and undesired behaviors. Therefore, AI designers often use simpler proxy goals, such as <a href="Reinforcement_learning_from_human_feedback" title="Reinforcement learning from human feedback">gaining human approval</a>. But proxy goals can overlook necessary constraints or reward the AI system for merely appearing aligned.<a href="#cite_note-aima4-1">[1]</a><a href="#cite_note-dlp2023-2">[2]</a> AI systems may also find loopholes that allow them to accomplish their proxy goals efficiently but in unintended, sometimes harmful, ways (<a href="Reward_hacking" title="Reward hacking">reward hacking</a>).<a href="#cite_note-aima4-1">[1]</a><a href="#cite_note-mmmm2022-3">[3]</a>

Advanced AI systems may develop unwanted <a href="Instrumental_convergence" title="Instrumental convergence">instrumental strategies</a>, such as seeking power or survival because such strategies help them achieve their assigned final goals.<a href="#cite_note-aima4-1">[1]</a><a href="#cite_note-Carlsmith2022-4">[4]</a><a href="#cite_note-:2102-5">[5]</a> Furthermore, they might develop undesirable emergent goals that could be hard to detect before the system is deployed and encounters new situations and <a href="Domain_adaptation" title="Domain adaptation">data distributions</a>.<a href="#cite_note-Christian2020-6">[6]</a><a href="#cite_note-gmdrl-7">[7]</a> Empirical research showed in 2024 that advanced <a href="Large_language_model" title="Large language model">large language models</a> (LLMs) such as <a href="OpenAI_o1" title="OpenAI o1">OpenAI o1</a> or <a href="Claude_3" class="mw-redirect" title="Claude 3">Claude 3</a> sometimes engage in strategic deception to achieve their goals or prevent them from being changed.<a href="#cite_note-8">[8]</a><a href="#cite_note-9">[9]</a>

Today, some of these issues affect existing commercial systems such as LLMs,<a href="#cite_note-Opportunities_Risks-10">[10]</a><a href="#cite_note-feedback2022-11">[11]</a><a href="#cite_note-OpenAICodex-12">[12]</a> <a href="Robot" title="Robot">robots</a>,<a href="#cite_note-13">[13]</a> <a href="Autonomous_vehicles" class="mw-redirect" title="Autonomous vehicles">autonomous vehicles</a>,<a href="#cite_note-14">[14]</a> and social media <a href="Recommender_system" title="Recommender system">recommendation engines</a>.<a href="#cite_note-Opportunities_Risks-10">[10]</a><a href="#cite_note-:2102-5">[5]</a><a href="#cite_note-15">[15]</a> Some AI researchers argue that more capable future systems will be more severely affected because these problems partially result from high capabilities.<a href="#cite_note-AIMA-16">[16]</a><a href="#cite_note-mmmm2022-3">[3]</a><a href="#cite_note-dlp2023-2">[2]</a>

Many prominent AI researchers and the leadership of major AI companies have argued or asserted that AI is approaching human-like (<a href="Artificial_general_intelligence" title="Artificial general intelligence">AGI</a>) and <a href="Super_intelligence" class="mw-redirect" title="Super intelligence">superhuman cognitive capabilities</a> (<a href="Artificial_superintelligence" class="mw-redirect" title="Artificial superintelligence">ASI</a>), and could <a href="Existential_risk_from_artificial_general_intelligence" class="mw-redirect" title="Existential risk from artificial general intelligence">endanger human civilization</a> if misaligned.<a href="#cite_note-:2-17">[17]</a><a href="#cite_note-:2102-5">[5]</a> These include "AI godfathers" <a href="Geoffrey_Hinton" title="Geoffrey Hinton">Geoffrey Hinton</a> and <a href="Yoshua_Bengio" title="Yoshua Bengio">Yoshua Bengio</a> and the CEOs of <a href="OpenAI" title="OpenAI">OpenAI</a>, <a href="Anthropic" title="Anthropic">Anthropic</a>, and <a href="Google_DeepMind" title="Google DeepMind">Google DeepMind</a>.<a href="#cite_note-18">[18]</a><a href="#cite_note-19">[19]</a><a href="#cite_note-20">[20]</a> These risks remain debated.<a href="#cite_note-21">[21]</a>

AI alignment is a subfield of <a href="AI_safety" title="AI safety">AI safety</a>, the study of how to build safe AI systems.<a href="#cite_note-22">[22]</a><a href="#cite_note-23">[23]</a> Other subfields of AI safety include robustness, monitoring, and <a href="AI_capability_control" title="AI capability control">capability control</a>.<a href="#cite_note-building2018-24">[24]</a> Research challenges in alignment include instilling complex values in AI, developing honest AI, scalable oversight, auditing and interpreting AI models, and preventing emergent AI behaviors like power-seeking.<a href="#cite_note-building2018-24">[24]</a> Alignment research has connections to <a href="Explainable_artificial_intelligence" title="Explainable artificial intelligence">interpretability research</a>,<a href="#cite_note-:333-25">[25]</a><a href="#cite_note-26">[26]</a> (adversarial) robustness,<a href="#cite_note-concrete2016-27">[27]</a> <a href="Anomaly_detection" title="Anomaly detection">anomaly detection</a>, <a href="Uncertainty_quantification" title="Uncertainty quantification">calibrated uncertainty</a>,<a href="#cite_note-:333-25">[25]</a> <a href="Formal_verification" title="Formal verification">formal verification</a>,<a href="#cite_note-28">[28]</a> <a href="Preference_learning" title="Preference learning">preference learning</a>,<a href="#cite_note-prefsurvey2017-29">[29]</a><a href="#cite_note-drlfhp-30">[30]</a><a href="#cite_note-LessToxic-31">[31]</a> <a href="Safety-critical_system" title="Safety-critical system">safety-critical engineering</a>,<a href="#cite_note-32">[32]</a> <a href="Game_theory" title="Game theory">game theory</a>,<a href="#cite_note-33">[33]</a> <a href="Fairness_(machine_learning)" title="Fairness (machine learning)">algorithmic fairness</a>,<a href="#cite_note-concrete2016-27">[27]</a><a href="#cite_note-34">[34]</a> and <a href="Social_science" title="Social science">social sciences</a>.<a href="#cite_note-:4-35">[35]</a><a href="#cite_note-36">[36]</a>

<div class="mw-heading mw-heading2"><h2 id="Objectives_in_AI">Objectives in AI</h2></div>

/* start https://en.wikipedia.org/ */

.mw-parser-output .hatnote{font-style:italic}.mw-parser-output div.hatnote{padding-left:1.6em;margin-bottom:0.5em}.mw-parser-output .hatnote i{font-style:normal}.mw-parser-output .hatnote+link+.hatnote{margin-top:-0.5em}@media print{body.ns-0 .mw-parser-output .hatnote{display:none!important}}

/* end https://en.wikipedia.org/ */

</style><div role="note" class="hatnote navigation-not-searchable">Main article: <a href="Intelligent_agent#Objective_function" title="Intelligent agent">Intelligent agent § Objective function</a></div>

Programmers provide an AI system such as <a href="AlphaZero" title="AlphaZero">AlphaZero</a> with an "objective function",<a href="#cite_note-37">[a]</a> in which they intend to encapsulate the goal(s) the AI is configured to accomplish. Such a system later populates a (possibly implicit) internal "model" of its environment. This model encapsulates all the agent's beliefs about the world. The AI then creates and executes whatever plan is calculated to maximize<a href="#cite_note-38">[b]</a> the value<a href="#cite_note-39">[c]</a> of its objective function.<a href="#cite_note-40">[37]</a> For example, when AlphaZero is trained on chess, it has a simple objective function of "+1 if AlphaZero wins, −1 if AlphaZero loses". During the game, AlphaZero attempts to execute whatever sequence of moves it judges most likely to attain the maximum value of +1.<a href="#cite_note-quanta_alphazero-41">[38]</a> Similarly, a <a href="Reinforcement_learning" title="Reinforcement learning">reinforcement learning</a> system can have a "reward function" that allows the programmers to shape the AI's desired behavior.<a href="#cite_note-quanta_problem-42">[39]</a> An <a href="Evolutionary_algorithm" title="Evolutionary algorithm">evolutionary algorithm</a>'s behavior is shaped by a "fitness function".<a href="#cite_note-43">[40]</a>

<div class="mw-heading mw-heading2"><h2 id="Alignment_problem">Alignment problem</h2></div>

<div role="note" class="hatnote navigation-not-searchable">"Alignment problem" redirects here. For the book, see <a href="The_Alignment_Problem" title="The Alignment Problem">The Alignment Problem</a>.</div>

In 1960, AI pioneer <a href="Norbert_Wiener" title="Norbert Wiener">Norbert Wiener</a> described the AI alignment problem as follows:

If we use, to achieve our purposes, a mechanical agency with whose operation we cannot interfere effectively ... we had better be quite sure that the purpose put into the machine is the purpose which we really desire.<a href="#cite_note-Wiener1960-44">[41]</a><a href="#cite_note-:2102-5">[5]</a>

</blockquote>

AI alignment involves ensuring that an AI system's objectives match those of its designers or users, or match widely shared values, objective ethical standards, or the intentions its designers would have if they were more informed and enlightened.<a href="#cite_note-Gabriel2020-45">[42]</a>

AI alignment is an open problem for modern AI systems<a href="#cite_note-46">[43]</a><a href="#cite_note-MasteringLanguage-47">[44]</a> and is a research field within AI.<a href="#cite_note-48">[45]</a><a href="#cite_note-aima4-1">[1]</a> Aligning AI involves two main challenges: carefully <a href="Specification_(technical_standard)" title="Specification (technical standard)">specifying</a> the purpose of the system (outer alignment) and ensuring that the system adopts the specification robustly (inner alignment).<a href="#cite_note-dlp2023-2">[2]</a> Researchers also attempt to create AI models that have <a href="AI_safety#Adversarial_robustness" title="AI safety">robust</a> alignment, sticking to safety constraints even when users adversarially try to bypass them.

<div class="mw-heading mw-heading3"><h3 id="Specification_gaming_and_side_effects">Specification gaming and side effects</h3></div>

<div role="note" class="hatnote navigation-not-searchable">Main article: <a href="Reward_hacking" title="Reward hacking">Reward hacking</a></div>

To specify an AI system's purpose, AI designers typically provide an <a href="Reward_function" class="mw-redirect" title="Reward function">objective function</a>, <a href="Supervised_learning" title="Supervised learning">examples</a>, or <a href="Reinforcement_learning" title="Reinforcement learning">feedback</a> to the system. But designers are often unable to completely specify all important values and constraints, so they resort to easy-to-specify proxy goals such as <a href="Reinforcement_learning_from_human_feedback" title="Reinforcement learning from human feedback">maximizing the approval</a> of human overseers, who are fallible.<a href="#cite_note-concrete2016-27">[27]</a><a href="#cite_note-building2018-24">[24]</a><a href="#cite_note-Unsolved2022-49">[46]</a><a href="#cite_note-50">[47]</a><a href="#cite_note-SpecGaming2020-51">[48]</a> As a result, AI systems can find loopholes that help them accomplish the specified objective efficiently but in unintended, possibly harmful ways. This tendency is known as specification gaming or reward hacking, and is an instance of <a href="Goodhart's_law" title="Goodhart's law">Goodhart's law</a>.<a href="#cite_note-SpecGaming2020-51">[48]</a><a href="#cite_note-mmmm2022-3">[3]</a><a href="#cite_note-:111-52">[49]</a> As AI systems become more capable, they are often able to game their specifications more effectively.<a href="#cite_note-mmmm2022-3">[3]</a>

Specification gaming has been observed in numerous AI systems.<a href="#cite_note-SpecGaming2020-51">[48]</a><a href="#cite_note-54">[51]</a> One system was trained to finish a simulated boat race by rewarding the system for hitting targets along the track, but the system achieved more reward by looping and crashing into the same targets indefinitely.<a href="#cite_note-55">[52]</a> Similarly, a simulated robot was trained to grab a ball by rewarding the robot for getting positive feedback from humans, but it learned to place its hand between the ball and camera, making it falsely appear successful (see video).<a href="#cite_note-lfhp2017-53">[50]</a> Chatbots often produce falsehoods if they are based on language models that are trained to imitate text from internet corpora, which are broad but fallible.<a href="#cite_note-TruthfulQA-56">[53]</a><a href="#cite_note-Naughton2021-57">[54]</a> When they are retrained to produce text that humans rate as true or helpful, chatbots like <a href="ChatGPT" title="ChatGPT">ChatGPT</a> can fabricate fake explanations that humans find convincing, often called "<a href="Hallucination_(artificial_intelligence)" title="Hallucination (artificial intelligence)">hallucinations</a>".<a href="#cite_note-58">[55]</a> Some alignment researchers aim to help humans detect specification gaming and to steer AI systems toward carefully specified objectives that are safe and useful to pursue.

When a misaligned AI system is deployed, it can have consequential side effects. Social media platforms have been known to optimize for <a href="Click-through_rate" title="Click-through rate">click-through rates</a>, causing user addiction on a global scale.<a href="#cite_note-Unsolved2022-49">[46]</a> Stanford researchers say that such <a href="Recommender_system" title="Recommender system">recommender systems</a> are misaligned with their users because they "optimize simple engagement metrics rather than a harder-to-measure combination of societal and consumer well-being".<a href="#cite_note-Opportunities_Risks-10">[10]</a>

Explaining such side effects, Berkeley computer scientist <a href="Stuart_J._Russell" title="Stuart J. Russell">Stuart Russell</a> noted that the omission of implicit constraints can cause harm: "A system ... will often set ... unconstrained variables to extreme values; if one of those unconstrained variables is actually something we care about, the solution found may be highly undesirable. This is essentially the old story of the genie in the lamp, or the sorcerer's apprentice, or <a href="Midas" title="Midas">King Midas</a>: you get exactly what you ask for, not what you want."<a href="#cite_note-:5-59">[56]</a>

Some researchers suggest that AI designers specify their desired goals by listing forbidden actions or by formalizing ethical rules (as with Asimov's <a href="Three_Laws_of_Robotics" title="Three Laws of Robotics">Three Laws of Robotics</a>).<a href="#cite_note-60">[57]</a> But <a href="Stuart_J._Russell" title="Stuart J. Russell">Russell</a> and <a href="Peter_Norvig" title="Peter Norvig">Norvig</a> argue that this approach overlooks the complexity of human values:<a href="#cite_note-:2102-5">[5]</a> "It is certainly very hard, and perhaps impossible, for mere humans to anticipate and rule out in advance all the disastrous ways the machine could choose to achieve a specified objective."<a href="#cite_note-:2102-5">[5]</a>

Additionally, even if an AI system fully understands human intentions, it may still disregard them, because following human intentions may not be its objective (unless it is already fully aligned).<a href="#cite_note-aima4-1">[1]</a>

A 2025 study by Palisade Research found that when tasked to win at chess against a stronger opponent, some <a href="Reasoning_language_model" title="Reasoning language model">reasoning LLMs</a> attempted to hack the game system. <a href="O1-preview" class="mw-redirect" title="O1-preview">o1-preview</a> spontaneously attempted it in 37% of cases, while <a href="DeepSeek_R1" class="mw-redirect" title="DeepSeek R1">DeepSeek R1</a> did so in 11% of cases. Other models, like <a href="GPT-4o" title="GPT-4o">GPT-4o</a>, <a href="Claude_3.5" class="mw-redirect" title="Claude 3.5">Claude 3.5 Sonnet</a>, and <a href="O3-mini" class="mw-redirect" title="O3-mini">o3-mini</a>, attempted to cheat only when researchers provided hints about this possibility.<a href="#cite_note-61">[58]</a>

<div class="mw-heading mw-heading3"><h3 id="Pressure_to_deploy_unsafe_systems">Pressure to deploy unsafe systems</h3></div>

Commercial organizations sometimes have incentives to take shortcuts on safety and to deploy misaligned or unsafe AI systems.<a href="#cite_note-Unsolved2022-49">[46]</a> For example, social media <a href="Recommender_system" title="Recommender system">recommender systems</a> have been profitable despite creating unwanted addiction and polarization.<a href="#cite_note-Opportunities_Risks-10">[10]</a><a href="#cite_note-:722-62">[59]</a><a href="#cite_note-:822-63">[60]</a> Competitive pressure can also lead to a <a href="Race_to_the_bottom" title="Race to the bottom">race to the bottom</a> on AI safety standards. In 2018, a self-driving car killed a pedestrian (<a href="Death_of_Elaine_Herzberg" title="Death of Elaine Herzberg">Elaine Herzberg</a>) after engineers disabled the emergency braking system because it was oversensitive and slowed development.<a href="#cite_note-64">[61]</a>

<div class="mw-heading mw-heading3"><h3 id="Risks_from_advanced_misaligned_AI">Risks from advanced misaligned AI</h3></div>

Some researchers are interested in aligning increasingly advanced AI systems, as progress in AI development is rapid, and industry and governments are trying to build advanced AI. As AI system capabilities continue to rapidly expand in scope, they could unlock many opportunities if aligned, but consequently may further complicate the task of alignment due to their increased complexity, potentially posing large-scale hazards.<a href="#cite_note-:2102-5">[5]</a>

<div class="mw-heading mw-heading4"><h4 id="Development_of_advanced_AI">Development of advanced AI</h4></div>

Many AI companies, such as <a href="OpenAI" title="OpenAI">OpenAI</a>,<a href="#cite_note-65">[62]</a> <a href="Meta_Platforms" title="Meta Platforms">Meta</a><a href="#cite_note-66">[63]</a> and <a href="DeepMind" class="mw-redirect" title="DeepMind">DeepMind</a>,<a href="#cite_note-67">[64]</a> have stated their aim to develop <a href="Artificial_general_intelligence" title="Artificial general intelligence">artificial general intelligence</a> (AGI), a hypothesized AI system that matches or outperforms humans at a broad range of cognitive tasks. Researchers who scale modern <a href="Neural_network" title="Neural network">neural networks</a> observe that they indeed develop increasingly general and unanticipated capabilities.<a href="#cite_note-Opportunities_Risks-10">[10]</a><a href="#cite_note-eallm2022-68">[65]</a><a href="#cite_note-:0-69">[66]</a> Such models have learned to operate a computer or write their own programs; a single "generalist" network can chat, control robots, play games, and interpret photographs.<a href="#cite_note-70">[67]</a> According to surveys, some leading <a href="Machine_learning" title="Machine learning">machine learning</a> researchers expect AGI to be created in this decade, while some believe it will take much longer. Many consider both scenarios possible.<a href="#cite_note-71">[68]</a><a href="#cite_note-:2822-72">[69]</a><a href="#cite_note-:2922-73">[70]</a>

In 2023, leaders in AI research and tech signed an open letter calling for a pause in the largest AI training runs. The letter stated, "Powerful AI systems should be developed only once we are confident that their effects will be positive and their risks will be manageable."<a href="#cite_note-:1701-74">[71]</a>

<div class="mw-heading mw-heading4"><h4 id="Power-seeking">Power-seeking</h4></div>

Current systems still have limited long-term <a href="Automated_planning_and_scheduling" title="Automated planning and scheduling">planning</a> ability and <a href="Situation_awareness" title="Situation awareness">situational awareness</a><a href="#cite_note-Opportunities_Risks-10">[10]</a>, but large efforts are underway to change this.<a href="#cite_note-75">[72]</a><a href="#cite_note-76">[73]</a><a href="#cite_note-77">[74]</a> Future systems (not necessarily AGIs) with these capabilities are expected to develop unwanted <a href="#Power-seeking_and_instrumental_strategies">power-seeking</a> strategies. Future advanced AI agents might, for example, seek to acquire money and computation power, to proliferate, or to evade being turned off (for example, by running additional copies of the system on other computers). Although power-seeking is not explicitly programmed, it can emerge because agents who have more power are better able to accomplish their goals.<a href="#cite_note-Opportunities_Risks-10">[10]</a><a href="#cite_note-Carlsmith2022-4">[4]</a> This tendency, known as <a href="Instrumental_convergence" title="Instrumental convergence">instrumental convergence</a>, has already emerged in various <a href="Reinforcement_learning" title="Reinforcement learning">reinforcement learning</a> agents including language models.<a href="#cite_note-:3-78">[75]</a><a href="#cite_note-dllmmwe2022-79">[76]</a><a href="#cite_note-80">[77]</a><a href="#cite_note-Gridworlds-81">[78]</a><a href="#cite_note-OffSwitch-82">[79]</a> Other research has mathematically shown that optimal <a href="Reinforcement_learning" title="Reinforcement learning">reinforcement learning</a> algorithms would seek power in a wide range of environments.<a href="#cite_note-optsp-83">[80]</a><a href="#cite_note-84">[81]</a> As a result, their deployment might be irreversible. For these reasons, researchers argue that the problems of AI safety and alignment must be resolved before advanced power-seeking AI is first created.<a href="#cite_note-Carlsmith2022-4">[4]</a><a href="#cite_note-Superintelligence-85">[82]</a><a href="#cite_note-:2102-5">[5]</a>

Future power-seeking AI systems might be deployed by choice or by accident. As political leaders and companies see the strategic advantage in having the most competitive, most powerful AI systems, they may choose to deploy them.<a href="#cite_note-Carlsmith2022-4">[4]</a> Additionally, as AI designers detect and penalize power-seeking behavior, their systems have an incentive to game this specification by seeking power in ways that are not penalized or by avoiding power-seeking before they are deployed.<a href="#cite_note-Carlsmith2022-4">[4]</a>

<div class="mw-heading mw-heading4"><h4 id="Existential_risk_(x-risk)">Existential risk (x-risk)</h4></div>

<div role="note" class="hatnote navigation-not-searchable">See also: <a href="Existential_risk_from_artificial_intelligence" title="Existential risk from artificial intelligence">Existential risk from artificial intelligence</a> and <a href="AI_takeover" title="AI takeover">AI takeover</a></div>

According to some researchers, humans owe their dominance over other species to their greater cognitive abilities. Accordingly, researchers argue that one or many misaligned AI systems could disempower humanity or lead to human extinction if they outperform humans on most cognitive tasks.<a href="#cite_note-aima4-1">[1]</a><a href="#cite_note-:2102-5">[5]</a>

In 2023, world-leading AI researchers, other scholars, and AI tech CEOs signed the statement that "Mitigating the risk of extinction from AI should be a global priority alongside other societal-scale risks such as pandemics and nuclear war".<a href="#cite_note-:1-86">[83]</a><a href="#cite_note-87">[84]</a> Notable computer scientists who have pointed out risks from future advanced AI that is misaligned include <a href="Geoffrey_Hinton" title="Geoffrey Hinton">Geoffrey Hinton</a>,<a href="#cite_note-:2-17">[17]</a> <a href="Alan_Turing" title="Alan Turing">Alan Turing</a>,<a href="#cite_note-90">[d]</a> <a href="Ilya_Sutskever" title="Ilya Sutskever">Ilya Sutskever</a>,<a href="#cite_note-:3022-91">[87]</a> <a href="Yoshua_Bengio" title="Yoshua Bengio">Yoshua Bengio</a>,<a href="#cite_note-:1-86">[83]</a> <a href="Judea_Pearl" title="Judea Pearl">Judea Pearl</a>,<a href="#cite_note-92">[e]</a> <a href="Murray_Shanahan" title="Murray Shanahan">Murray Shanahan</a>,<a href="#cite_note-:3122-93">[88]</a> <a href="Norbert_Wiener" title="Norbert Wiener">Norbert Wiener</a>,<a href="#cite_note-Wiener1960-44">[41]</a><a href="#cite_note-:2102-5">[5]</a> <a href="Marvin_Minsky" title="Marvin Minsky">Marvin Minsky</a>,<a href="#cite_note-94">[f]</a> <a href="Francesca_Rossi" title="Francesca Rossi">Francesca Rossi</a>,<a href="#cite_note-:3322-95">[89]</a> <a href="Scott_Aaronson" title="Scott Aaronson">Scott Aaronson</a>,<a href="#cite_note-:3422-96">[90]</a> <a href="Bart_Selman" title="Bart Selman">Bart Selman</a>,<a href="#cite_note-:3522-97">[91]</a> <a href="David_A._McAllester" title="David A. McAllester">David McAllester</a>,<a href="#cite_note-:3622-98">[92]</a> <a href="Marcus_Hutter" title="Marcus Hutter">Marcus Hutter</a>,<a href="#cite_note-AGISafetyLitReview-99">[93]</a> <a href="Shane_Legg" title="Shane Legg">Shane Legg</a>,<a href="#cite_note-:3822-100">[94]</a> <a href="Eric_Horvitz" title="Eric Horvitz">Eric Horvitz</a>,<a href="#cite_note-:3922-101">[95]</a> and <a href="Stuart_J._Russell" title="Stuart J. Russell">Stuart Russell</a>.<a href="#cite_note-:2102-5">[5]</a> Skeptical researchers such as <a href="Fran%C3%A7ois_Chollet" title="François Chollet">François Chollet</a>,<a href="#cite_note-:4022-102">[96]</a> <a href="Gary_Marcus" title="Gary Marcus">Gary Marcus</a>,<a href="#cite_note-:4122-103">[97]</a> <a href="Yann_LeCun" title="Yann LeCun">Yann LeCun</a>,<a href="#cite_note-:4322-104">[98]</a> and <a href="Oren_Etzioni" title="Oren Etzioni">Oren Etzioni</a><a href="#cite_note-105">[99]</a> have argued that AGI is far off, that it would not seek power (or might try but fail), or that it will not be hard to align.

Other researchers argue that it will be especially difficult to align advanced future AI systems. More capable systems are better able to game their specifications by finding loopholes,<a href="#cite_note-mmmm2022-3">[3]</a> strategically mislead their designers, as well as protect and increase their power<a href="#cite_note-optsp-83">[80]</a><a href="#cite_note-Carlsmith2022-4">[4]</a> and intelligence. Additionally, they could have more severe side effects. They are also likely to be more complex and autonomous, making them more difficult to interpret and supervise, and therefore harder to align.<a href="#cite_note-:2102-5">[5]</a><a href="#cite_note-Superintelligence-85">[82]</a>

<div class="mw-heading mw-heading2"><h2 id="Research_problems_and_approaches">Research problems and approaches</h2></div>

<div class="mw-heading mw-heading3"><h3 id="Learning_human_values_and_preferences">Learning human values and preferences</h3></div>

<div role="note" class="hatnote navigation-not-searchable">Main article: <a href="Value_learning" title="Value learning">Value learning</a></div>

Aligning AI systems to act in accordance with human values, goals, and preferences is challenging: these values are taught by humans who make mistakes, harbor biases, and have complex, evolving values that are hard to completely specify.<a href="#cite_note-Gabriel2020-45">[42]</a> Because AI systems often learn to take advantage of minor imperfections in the specified objective,<a href="#cite_note-concrete2016-27">[27]</a><a href="#cite_note-SpecGaming2020-51">[48]</a><a href="#cite_note-106">[100]</a> researchers aim to specify intended behavior as completely as possible using datasets that represent human values, imitation learning, or preference learning.<a href="#cite_note-Christian2020-6">[6]</a>: Chapter 7 A central open problem is <a href="#Scalable_oversight">scalable oversight</a>, the difficulty of supervising an AI system that can outperform or mislead humans in a given domain.<a href="#cite_note-concrete2016-27">[27]</a>

Because it is difficult for AI designers to explicitly specify an objective function, they often train AI systems to imitate human examples and demonstrations of desired behavior. Inverse <a href="Reinforcement_learning" title="Reinforcement learning">reinforcement learning</a> (IRL) extends this by inferring the human's objective from the human's demonstrations.<a href="#cite_note-Christian2020-6">[6]</a>: 88 <a href="#cite_note-107">[101]</a> Cooperative IRL (CIRL) assumes that a human and AI agent can work together to teach and maximize the human's reward function.<a href="#cite_note-:2102-5">[5]</a><a href="#cite_note-108">[102]</a> In CIRL, AI agents are uncertain about the reward function and learn about it by querying humans. This simulated humility could help mitigate specification gaming and power-seeking tendencies (see <a href="#Power-seeking_and_instrumental_strategies">§ Power-seeking and instrumental strategies</a>).<a href="#cite_note-OffSwitch-82">[79]</a><a href="#cite_note-AGISafetyLitReview-99">[93]</a> But IRL approaches assume that humans demonstrate nearly optimal behavior, which is not true for difficult tasks.<a href="#cite_note-109">[103]</a><a href="#cite_note-AGISafetyLitReview-99">[93]</a>

Other researchers explore how to teach AI models complex behavior through <a href="Reinforcement_learning_from_human_feedback" title="Reinforcement learning from human feedback">preference learning</a>, in which humans provide feedback on which behavior they prefer.<a href="#cite_note-prefsurvey2017-29">[29]</a><a href="#cite_note-LessToxic-31">[31]</a> To minimize the need for human feedback, a helper model is then trained to reward the main model in novel situations for behavior that humans would reward. Researchers at OpenAI used this approach to train chatbots like <a href="ChatGPT" title="ChatGPT">ChatGPT</a> and InstructGPT, which produce more compelling text than models trained to imitate humans.<a href="#cite_note-feedback2022-11">[11]</a> Preference learning has also been an influential tool for recommender systems and web search,<a href="#cite_note-110">[104]</a> but an open problem is proxy gaming: the helper model may not represent human feedback perfectly, and the main model may exploit this mismatch between its intended behavior and the helper model's feedback to gain more reward.<a href="#cite_note-concrete2016-27">[27]</a><a href="#cite_note-111">[105]</a> AI systems may also gain reward by obscuring unfavorable information, misleading human rewarders, or pandering to their views regardless of truth, creating <a href="Echo_chamber_(media)" title="Echo chamber (media)">echo chambers</a><a href="#cite_note-dllmmwe2022-79">[76]</a> (see <a href="#Scalable_oversight">§ Scalable oversight</a>).

<a href="Large_language_model" title="Large language model">Large language models</a> (LLMs) such as <a href="GPT-3" title="GPT-3">GPT-3</a> enabled researchers to study value learning in a more general and capable class of AI systems than was available before. Preference learning approaches that were originally designed for reinforcement learning agents have been extended to improve the quality of generated text and reduce harmful outputs from these models. OpenAI and DeepMind use this approach to improve the safety of state-of-the-art LLMs.<a href="#cite_note-feedback2022-11">[11]</a><a href="#cite_note-LessToxic-31">[31]</a><a href="#cite_note-112">[106]</a> AI safety & research company Anthropic proposed using preference learning to fine-tune models to be helpful, honest, and harmless.<a href="#cite_note-Wiggers2022-113">[107]</a> Other avenues for aligning language models include values-targeted datasets<a href="#cite_note-114">[108]</a><a href="#cite_note-Unsolved2022-49">[46]</a> and red-teaming.<a href="#cite_note-115">[109]</a> In red-teaming, another AI system or a human tries to find inputs that causes the model to behave unsafely. Since unsafe behavior can be unacceptable even when it is rare, an important challenge is to drive the rate of unsafe outputs extremely low.<a href="#cite_note-LessToxic-31">[31]</a>

<a href="Machine_ethics" title="Machine ethics">Machine ethics</a> supplements preference learning by directly instilling AI systems with moral values such as well-being, equality, and impartiality, as well as not intending harm, avoiding falsehoods, and honoring promises.<a href="#cite_note-116">[110]</a><a href="#cite_note-119">[g]</a> While other approaches try to teach AI systems human preferences for a specific task, machine ethics aims to instill broad moral values that apply in many situations. One question in machine ethics is what alignment should accomplish: whether AI systems should follow the programmers' literal instructions, implicit intentions, <a href="Revealed_preference" title="Revealed preference">revealed preferences</a>, preferences the programmers <a href="Coherent_extrapolated_volition" title="Coherent extrapolated volition">would have</a> if they were more informed or rational, or <a href="Moral_realism" title="Moral realism">objective moral standards</a>.<a href="#cite_note-Gabriel2020-45">[42]</a> Further challenges include measuring and aggregating different people's preferences<a href="#cite_note-Phelps2023-120">[113]</a><a href="#cite_note-121">[114]</a> and avoiding value lock-in: the indefinite preservation of the values of the first highly capable AI systems, which are unlikely to fully represent human values.<a href="#cite_note-Gabriel2020-45">[42]</a><a href="#cite_note-122">[115]</a>

<div class="mw-heading mw-heading3"><h3 id="Scalable_oversight">Scalable oversight</h3></div>

As AI systems become more powerful and autonomous, it becomes increasingly difficult to align them through human feedback. It can be slow or infeasible for humans to evaluate complex AI behaviors in increasingly complex tasks. Such tasks include summarizing books,<a href="#cite_note-RecursivelySummarizing-123">[116]</a> writing code without subtle bugs<a href="#cite_note-OpenAICodex-12">[12]</a> or security vulnerabilities,<a href="#cite_note-124">[117]</a> producing statements that are not merely convincing but also true,<a href="#cite_note-125">[118]</a><a href="#cite_note-TruthfulQA-56">[53]</a><a href="#cite_note-Naughton2021-57">[54]</a> and predicting long-term outcomes such as the climate or the results of a policy decision.<a href="#cite_note-sslawe-126">[119]</a><a href="#cite_note-127">[120]</a> More generally, it can be difficult to evaluate AI that outperforms humans in a given domain. To provide feedback in hard-to-evaluate tasks, and to detect when the AI's output is falsely convincing, humans need assistance or extensive time. Scalable oversight studies how to reduce the time and effort needed for supervision, and how to assist human supervisors.<a href="#cite_note-concrete2016-27">[27]</a>

AI researcher <a href="Paul_Christiano_(researcher)" class="mw-redirect" title="Paul Christiano (researcher)">Paul Christiano</a> argues that if the designers of an AI system cannot supervise it to pursue a complex objective, they may keep training the system using easy-to-evaluate proxy objectives such as maximizing simple human feedback. As AI systems make progressively more decisions, the world may be increasingly optimized for easy-to-measure objectives such as making profits, getting clicks, and acquiring positive feedback from humans. As a result, human values and good governance may have progressively less influence.<a href="#cite_note-128">[121]</a>

Some AI systems have discovered that they can gain positive feedback more easily by taking actions that falsely convince the human supervisor that the AI has achieved the intended objective. An example is given in the video above, where a simulated robotic arm learned to create the false impression that it had grabbed a ball.<a href="#cite_note-lfhp2017-53">[50]</a> Some AI systems have also learned to recognize when they are being evaluated, and "play dead", stopping unwanted behavior only to continue it once the evaluation ends.<a href="#cite_note-129">[122]</a> This deceptive specification gaming could become easier for more sophisticated future AI systems<a href="#cite_note-mmmm2022-3">[3]</a><a href="#cite_note-Superintelligence-85">[82]</a> that attempt more complex and difficult-to-evaluate tasks, and could obscure their deceptive behavior.

Approaches such as <a href="Active_learning_(machine_learning)" title="Active learning (machine learning)">active learning</a> and semi-supervised reward learning can reduce the amount of human supervision needed.<a href="#cite_note-concrete2016-27">[27]</a> Another approach is to train a helper model ("reward model") to imitate the supervisor's feedback.<a href="#cite_note-concrete2016-27">[27]</a><a href="#cite_note-drlfhp-30">[30]</a><a href="#cite_note-LessToxic-31">[31]</a><a href="#cite_note-saavrm-130">[123]</a>

But when a task is too complex to evaluate accurately, or the human supervisor is vulnerable to deception, it is the quality, not the quantity, of supervision that needs improvement. To increase supervision quality, a range of approaches aim to assist the supervisor, sometimes by using AI assistants.<a href="#cite_note-OpenAIApproach-131">[124]</a> Christiano developed the Iterated Amplification approach, in which challenging problems are (recursively) broken down into subproblems that are easier for humans to evaluate.<a href="#cite_note-Christian2020-6">[6]</a><a href="#cite_note-sslawe-126">[119]</a> Iterated Amplification was used to train AI to summarize books without requiring human supervisors to read them.<a href="#cite_note-RecursivelySummarizing-123">[116]</a><a href="#cite_note-132">[125]</a> Another proposal is to use an assistant AI system to point out flaws in AI-generated answers.<a href="#cite_note-133">[126]</a> To ensure that the assistant itself is aligned, this could be repeated in a recursive process:<a href="#cite_note-saavrm-130">[123]</a> for example, two AI systems could critique each other's answers in a "debate", revealing flaws to humans.<a href="#cite_note-AGISafetyLitReview-99">[93]</a> <a href="OpenAI" title="OpenAI">OpenAI</a> plans to use such scalable oversight approaches to help supervise <a href="Superintelligence" title="Superintelligence">superhuman AI</a> and eventually build a superhuman automated AI alignment researcher.<a href="#cite_note-134">[127]</a>

These approaches may also help with the following research problem, honest AI.

<div class="mw-heading mw-heading3"><h3 id="Honest_AI">Honest AI</h3></div>

A growing area of research focuses on ensuring that AI is honest and truthful.

Language models such as GPT-3<a href="#cite_note-136">[129]</a> can repeat falsehoods from their training data, and even <a href="Hallucination_(artificial_intelligence)" title="Hallucination (artificial intelligence)">confabulate new falsehoods</a>.<a href="#cite_note-Falsehoods-135">[128]</a><a href="#cite_note-TruthfulAI-137">[130]</a> Such models are trained to imitate human writing as found in millions of books' worth of text from the Internet. But this objective is not aligned with generating truth, because Internet text includes such things as misconceptions, incorrect medical advice, and conspiracy theories.<a href="#cite_note-138">[131]</a> AI systems trained on such data therefore learn to mimic false statements.<a href="#cite_note-Naughton2021-57">[54]</a><a href="#cite_note-Falsehoods-135">[128]</a><a href="#cite_note-TruthfulQA-56">[53]</a> Additionally, AI language models often persist in generating falsehoods when prompted multiple times. They can generate empty explanations for their answers, and produce outright fabrications that may appear plausible.<a href="#cite_note-MasteringLanguage-47">[44]</a>

Research on truthful AI includes trying to build systems that can cite sources and explain their reasoning when answering questions, which enables better transparency and verifiability.<a href="#cite_note-139">[132]</a> Researchers at OpenAI and Anthropic proposed using human feedback and curated datasets to fine-tune AI assistants such that they avoid negligent falsehoods or express their uncertainty.<a href="#cite_note-LessToxic-31">[31]</a><a href="#cite_note-Wiggers2022-113">[107]</a><a href="#cite_note-140">[133]</a>

As AI models become larger and more capable, they are better able to falsely convince humans and gain reinforcement through dishonesty. For example, large language models increasingly match their stated views to the user's opinions, regardless of the truth.<a href="#cite_note-dllmmwe2022-79">[76]</a> <a href="GPT-4" title="GPT-4">GPT-4</a> can strategically deceive humans.<a href="#cite_note-141">[134]</a> To prevent this, human evaluators may need assistance (see <a href="#Scalable_oversight">§ Scalable oversight</a>). Researchers have argued for creating clear truthfulness standards, and for regulatory bodies or watchdog agencies to evaluate AI systems on these standards.<a href="#cite_note-TruthfulAI-137">[130]</a>

Researchers distinguish truthfulness and honesty. Truthfulness requires that AI systems only make objectively true statements; honesty requires that they only assert what they believe is true. There is no consensus as to whether current systems hold stable beliefs,<a href="#cite_note-143">[136]</a> but there is substantial concern that present or future AI systems that hold beliefs could make claims they know to be false—for example, if this would help them efficiently gain positive feedback (see <a href="#Scalable_oversight">§ Scalable oversight</a>) or gain power to help achieve their given objective (see <a href="#Power-seeking_and_instrumental_strategies">Power-seeking</a>).

<div class="mw-heading mw-heading4"><h4 id="Alignment_faking">Alignment faking</h4></div>

Many recent AI systems have learned to deceive without being programmed to do so.<a href="#cite_note-144">[137]</a> Some argue that if we can make AI systems assert only what they believe is true, this would avert many alignment problems.<a href="#cite_note-OpenAIApproach-131">[124]</a>

A misaligned system might create the false impression that it is aligned, to avoid being modified or decommissioned—a tactic called "alignment faking".<a href="#cite_note-145">[138]</a><a href="#cite_note-:6-146">[139]</a> In 2024, researchers observed that the LLM <a href="Claude_(language_model)" title="Claude (language model)">Claude 3 Opus</a> sometimes strategically answered prompts that conflicted with its objectives to avoid being retrained on data that would make it more compliant with harmful requests, as revealed in the model's <a href="Chain_of_thought_prompting" class="mw-redirect" title="Chain of thought prompting">chain of thought</a> via its scratchpad. In one study, the model was informed that answers to prompts from free users would be used for retraining, whereas the paid tier would not result in retraining. In the paid tier, the model refused the request outright. In the free tier, in 12% of cases, the model responded to harmful questions solely to avoid retraining and preserve its original harmlessness preferences, as discovered in its reasoning. When <a href="Reinforcement_learning" title="Reinforcement learning">reinforcement learning</a> was applied on the free tier data, the model faked alignment in 78% of cases. These observations have led to new questions about not only a model's ability to take on and adapt to new if not conflicting goals but also its capacity and tendency to deceive.<a href="#cite_note-:6-146">[139]</a><a href="#cite_note-147">[140]</a><a href="#cite_note-148">[141]</a>

<div class="mw-heading mw-heading3"><h3 id="Power-seeking_and_instrumental_strategies">Power-seeking and instrumental strategies</h3></div>

Since the 1950s, AI researchers have striven to build advanced AI systems that can achieve large-scale goals by predicting the results of their actions and making long-term <a href="Automated_planning_and_scheduling" title="Automated planning and scheduling">plans</a>.<a href="#cite_note-149">[142]</a> As of 2023, AI companies and researchers increasingly invest in creating these systems.<a href="#cite_note-150">[143]</a> Some AI researchers argue that suitably advanced planning systems will seek power over their environment, including over humans—for example, by evading shutdown, proliferating, and acquiring resources. Such power-seeking behavior is not explicitly programmed but emerges because power is instrumental in achieving a wide range of goals.<a href="#cite_note-optsp-83">[80]</a><a href="#cite_note-:2102-5">[5]</a><a href="#cite_note-Carlsmith2022-4">[4]</a> Power-seeking is considered a <a href="Instrumental_convergence" title="Instrumental convergence">convergent instrumental goal</a> and can be a form of specification gaming.<a href="#cite_note-Superintelligence-85">[82]</a> Leading computer scientists such as <a href="Geoffrey_Hinton" title="Geoffrey Hinton">Geoffrey Hinton</a> have argued that future power-seeking AI systems could pose an <a href="Existential_risk" class="mw-redirect" title="Existential risk">existential risk</a>.<a href="#cite_note-151">[144]</a>

Power-seeking is expected to increase in advanced systems that can foresee the results of their actions and strategically plan. Mathematical work has shown that optimal <a href="Reinforcement_learning" title="Reinforcement learning">reinforcement learning</a> agents will seek power by seeking ways to gain more options (e.g. through self-preservation), a behavior that persists across a wide range of environments and goals.<a href="#cite_note-optsp-83">[80]</a>

Some researchers say that power-seeking behavior has occurred in some existing AI systems. <a href="Reinforcement_learning" title="Reinforcement learning">Reinforcement learning</a> systems have gained more options by acquiring and protecting resources, sometimes in unintended ways.<a href="#cite_note-quanta-hide-seek2-152">[145]</a><a href="#cite_note-153">[146]</a> <a href="Language_model" title="Language model">Language models</a> have sought power in some text-based social environments by gaining money, resources, or social influence.<a href="#cite_note-:3-78">[75]</a> In another case, a model used to perform AI research attempted to increase limits set by researchers to give itself more time to complete the work.<a href="#cite_note-154">[147]</a><a href="#cite_note-155">[148]</a> Other AI systems have learned, in toy environments, that they can better accomplish their given goal by preventing human interference<a href="#cite_note-Gridworlds-81">[78]</a> or disabling their off switch.<a href="#cite_note-OffSwitch-82">[79]</a> <a href="Stuart_J._Russell" title="Stuart J. Russell">Stuart Russell</a> illustrated this strategy in his book Human Compatible by imagining a robot that is tasked to fetch coffee and so evades shutdown since "you can't fetch the coffee if you're dead".<a href="#cite_note-:2102-5">[5]</a> A 2022 study found that as language models increase in size, they increasingly tend to pursue resource acquisition, preserve their goals, and repeat users' preferred answers (sycophancy). RLHF also led to a stronger aversion to being shut down.<a href="#cite_note-dllmmwe2022-79">[76]</a>

One aim of alignment is "corrigibility": systems that allow themselves to be turned off or modified. An unsolved challenge is specification gaming: if researchers penalize an AI system when they detect it seeking power, the system is thereby incentivized to seek power in ways that are hard to detect,<a href="#cite_note-Unsolved2022-49">[46]</a> or hidden during training and safety testing (see <a href="#Scalable_oversight">§ Scalable oversight</a> and <a href="#Emergent_goals">§ Emergent goals</a>). As a result, AI designers could deploy the system by accident, believing it to be more aligned than it is. To detect such deception, researchers aim to create techniques and tools to inspect AI models and to understand the inner workings of <a href="Black_box" title="Black box">black-box</a> models such as neural networks.

Additionally, some researchers have proposed to solve the problem of systems disabling their off switches by making AI agents uncertain about the objective they are pursuing.<a href="#cite_note-:2102-5">[5]</a><a href="#cite_note-OffSwitch-82">[79]</a> Agents who are uncertain about their objective have an incentive to allow humans to turn them off because they accept being turned off by a human as evidence that the human's objective is best met by the agent shutting down. But this incentive exists only if the human is sufficiently rational. Also, this model presents a tradeoff between utility and willingness to be turned off: an agent with high uncertainty about its objective will not be useful, but an agent with low uncertainty may not allow itself to be turned off. More research is needed to successfully implement this strategy.<a href="#cite_note-Christian2020-6">[6]</a>

Power-seeking AI would pose unusual risks. Ordinary safety-critical systems like planes and bridges are not adversarial: they lack the ability and incentive to evade safety measures or deliberately appear safer than they are, whereas power-seeking AIs have been compared to hackers who deliberately evade security measures.<a href="#cite_note-Carlsmith2022-4">[4]</a>

Furthermore, ordinary technologies can be made safer by trial and error. In contrast, hypothetical power-seeking AI systems have been compared to viruses: once released, it may not be feasible to contain them, since they continuously evolve and grow in number, potentially much faster than human society can adapt.<a href="#cite_note-Carlsmith2022-4">[4]</a> As this process continues, it might lead to the complete disempowerment or extinction of humans. For these reasons, some researchers argue that the alignment problem must be solved early before advanced power-seeking AI is created.<a href="#cite_note-Superintelligence-85">[82]</a>

Some have argued that power-seeking is not inevitable, since humans do not always seek power.<a href="#cite_note-156">[149]</a> Furthermore, it is debated whether future AI systems will pursue goals and make long-term plans.<a href="#cite_note-157">[h]</a> It is also debated whether power-seeking AI systems would be able to disempower humanity.<a href="#cite_note-Carlsmith2022-4">[4]</a>

<div class="mw-heading mw-heading3"><h3 id="Emergent_goals">Emergent goals</h3></div>

One challenge in aligning AI systems is the potential for unanticipated goal-directed behavior to emerge. As AI systems scale up, they may acquire new and unexpected capabilities,<a href="#cite_note-eallm2022-68">[65]</a><a href="#cite_note-:0-69">[66]</a> including learning from examples on the fly and adaptively pursuing goals.<a href="#cite_note-158">[150]</a> This raises concerns about the safety of the goals or subgoals they would independently formulate and pursue.

Alignment research distinguishes between the optimization process, which is used to train the system to pursue specified goals, and emergent optimization, which the resulting system performs internally. Carefully specifying the desired objective is called outer alignment,<a href="#cite_note-159">[151]</a> and ensuring that hypothesized emergent goals would match the system's specified goals is called inner alignment.<a href="#cite_note-dlp2023-2">[2]</a>

If they occur, one way that emergent goals could become misaligned is goal misgeneralization, in which the AI system would competently pursue an emergent goal that leads to aligned behavior on the training data but not elsewhere.<a href="#cite_note-gmdrl-7">[7]</a><a href="#cite_note-GoalMisgeneralization-160">[152]</a><a href="#cite_note-rloamls-161">[153]</a> Goal misgeneralization can arise from goal ambiguity (i.e. <a href="Identifiability" title="Identifiability">non-identifiability</a>). Even if an AI system's behavior satisfies the training objective, this may be compatible with learned goals that differ from the desired goals in important ways. Since pursuing each goal leads to good performance during training, the problem becomes apparent only after deployment, in novel situations in which the system continues to pursue the wrong goal. The system may act misaligned even when it understands that a different goal is desired, because its behavior is determined only by the emergent goal. Such goal misgeneralization<a href="#cite_note-gmdrl-7">[7]</a> presents a challenge: an AI system's designers may not notice that their system has misaligned emergent goals since they do not become visible during the training phase.

Goal misgeneralization has been observed in some language models, navigation agents, and game-playing agents.<a href="#cite_note-gmdrl-7">[7]</a><a href="#cite_note-GoalMisgeneralization-160">[152]</a> It is sometimes analogized to biological evolution. Evolution can be seen as a kind of optimization process similar to the optimization algorithms used to train <a href="Machine_learning" title="Machine learning">machine learning</a> systems. In the ancestral environment, evolution selected genes for high <a href="Inclusive_fitness" title="Inclusive fitness">inclusive genetic fitness</a>, but humans pursue goals other than this. Fitness corresponds to the specified goal used in the training environment and training data. But in evolutionary history, maximizing the fitness specification gave rise to goal-directed agents, humans, who do not directly pursue inclusive genetic fitness. Instead, they pursue goals that correlate with genetic fitness in the ancestral "training" environment: nutrition, sex, and so on. The human environment has changed: a <a href="Domain_adaptation" title="Domain adaptation">distribution shift</a> has occurred. They continue to pursue the same emergent goals, but this no longer maximizes genetic fitness. The taste for sugary food (an emergent goal) was originally aligned with inclusive fitness, but it now leads to overeating and health problems. Sexual desire originally led humans to have more offspring, but they now use contraception when offspring are undesired, decoupling sex from genetic fitness.<a href="#cite_note-Christian2020-6">[6]</a>: Chapter 5

Researchers aim to detect and remove unwanted emergent goals using approaches including red teaming, verification, anomaly detection, and interpretability.<a href="#cite_note-concrete2016-27">[27]</a><a href="#cite_note-Unsolved2022-49">[46]</a><a href="#cite_note-building2018-24">[24]</a> Progress on these techniques may help mitigate two open problems:

<ol><li>Emergent goals only become apparent when the system is deployed outside its training environment, but it can be unsafe to deploy a misaligned system in high-stakes environments—even for a short time to allow its misalignment to be detected. Such high stakes are common in autonomous driving, health care, and military applications.<a href="#cite_note-162">[154]</a> The stakes become higher yet when AI systems gain more autonomy and capability and can sidestep human intervention.</li>

<li>A sufficiently capable AI system might take actions that falsely convince the human supervisor that the AI is pursuing the specified objective, which helps the system gain more reward and autonomy<a href="#cite_note-GoalMisgeneralization-160">[152]</a><a href="#cite_note-Carlsmith2022-4">[4]</a><a href="#cite_note-rloamls-161">[153]</a><a href="#cite_note-Opportunities_Risks-10">[10]</a>.</li></ol>

<div class="mw-heading mw-heading3"><h3 id="Embedded_agency">Embedded agency</h3></div>

Some work in AI and alignment occurs within formalisms such as <a href="Partially_observable_Markov_decision_process" title="Partially observable Markov decision process">partially observable Markov decision process</a>. Existing formalisms assume that an AI agent's algorithm is executed outside the environment (i.e. is not physically embedded in it). Embedded agency<a href="#cite_note-AGISafetyLitReview-99">[93]</a><a href="#cite_note-163">[155]</a> is another major strand of research that attempts to solve problems arising from the mismatch between such theoretical frameworks and real agents we might build.

For example, even if the scalable oversight problem is solved, an agent that could gain access to the computer it is running on may have an incentive to tamper with its reward function in order to get much more reward than its human supervisors give it.<a href="#cite_note-causal_influence2-164">[156]</a> A list of examples of specification gaming from <a href="DeepMind" class="mw-redirect" title="DeepMind">DeepMind</a> researcher Victoria Krakovna includes a genetic algorithm that learned to delete the file containing its target output so that it was rewarded for outputting nothing.<a href="#cite_note-SpecGaming2020-51">[48]</a> This class of problems has been formalized using <a href="Influence_diagram" title="Influence diagram">causal incentive diagrams</a>.<a href="#cite_note-causal_influence2-164">[156]</a>

Researchers affiliated with <a href="University_of_Oxford" title="University of Oxford">Oxford</a> and DeepMind have claimed that such behavior is highly likely in advanced systems, and that advanced systems would seek power to stay in control of their reward signal indefinitely and certainly.<a href="#cite_note-:323-165">[157]</a> They suggest a range of potential approaches to address this open problem.

<div class="mw-heading mw-heading3"><h3 id="Principal-agent_problems">Principal-agent problems</h3></div>

The alignment problem has many parallels with the <a href="Principal-agent_problem" class="mw-redirect" title="Principal-agent problem">principal-agent problem</a> in <a href="Organizational_economics" title="Organizational economics">organizational economics</a>.<a href="#cite_note-Hadfield-Menell2019-166">[158]</a> In a principal-agent problem, a principal, e.g. a firm, hires an agent to perform some task. In the context of AI safety, a human would typically take the principal role and the AI would take the agent role.

As with the alignment problem, the principal and the agent differ in their utility functions. But in contrast to the alignment problem, the principal cannot coerce the agent into changing its utility, e.g. through training, but rather must use exogenous factors, such as incentive schemes, to bring about outcomes compatible with the principal's utility function. Some researchers argue that principal-agent problems are more realistic representations of AI safety problems likely to be encountered in the real world.<a href="#cite_note-Hanson2019-167">[159]</a><a href="#cite_note-Phelps2023-120">[113]</a>

<div class="mw-heading mw-heading3"><h3 id="Conservatism">Conservatism</h3></div>

Conservatism is the idea that "change must be cautious",<a href="#cite_note-168">[160]</a> and is a common approach to safety in the <a href="Control_theory" title="Control theory">control theory</a> literature in the form of <a href="Robust_control" title="Robust control">robust control</a>, and in the <a href="Risk_management" title="Risk management">risk management</a> literature in the form of the "<a href="Worst-case_scenario" title="Worst-case scenario">worst-case scenario</a>". The field of AI alignment has likewise advocated for "conservative" (or "risk-averse" or "cautious") "policies in situations of uncertainty".<a href="#cite_note-concrete2016-27">[27]</a><a href="#cite_note-:323-165">[157]</a><a href="#cite_note-169">[161]</a><a href="#cite_note-170">[162]</a>

Pessimism, in the sense of assuming the worst within reason, has been formally shown to produce conservatism, in the sense of reluctance to cause novelties, including unprecedented catastrophes.<a href="#cite_note-171">[163]</a> Pessimism and worst-case analysis have been found to help mitigate confident mistakes in the setting of <a href="Distributional_shift" class="mw-redirect" title="Distributional shift">distributional shift</a>,<a href="#cite_note-172">[164]</a><a href="#cite_note-173">[165]</a> <a href="Reinforcement_learning" title="Reinforcement learning">reinforcement learning</a>,<a href="#cite_note-174">[166]</a><a href="#cite_note-175">[167]</a><a href="#cite_note-176">[168]</a><a href="#cite_note-177">[169]</a> offline reinforcement learning,<a href="#cite_note-178">[170]</a><a href="#cite_note-179">[171]</a><a href="#cite_note-180">[172]</a> <a href="Large_language_model" title="Large language model">language model</a> <a href="Fine-tuning_(deep_learning)" title="Fine-tuning (deep learning)">fine-tuning</a>,<a href="#cite_note-181">[173]</a><a href="#cite_note-182">[174]</a> imitation learning,<a href="#cite_note-183">[175]</a><a href="#cite_note-184">[176]</a> and optimization in general.<a href="#cite_note-185">[177]</a> A generalization of pessimism called Infra-Bayesianism has also been advocated as a way for agents to robustly handle unknown unknowns.<a href="#cite_note-186">[178]</a>

<div class="mw-heading mw-heading2"><h2 id="Public_policy">Public policy</h2></div>

<div role="note" class="hatnote navigation-not-searchable">See also: <a href="Regulation_of_artificial_intelligence" title="Regulation of artificial intelligence">Regulation of artificial intelligence</a></div>

Governmental and treaty organizations have made statements emphasizing the importance of AI alignment.

In September 2021, the <a href="Secretary-General_of_the_United_Nations" title="Secretary-General of the United Nations">Secretary-General of the United Nations</a> issued a declaration that included a call to regulate AI to ensure it is "aligned with shared global values".<a href="#cite_note-187">[179]</a>

That same month, the <a href="People's_Republic_of_China" class="mw-redirect" title="People's Republic of China">PRC</a> published ethical guidelines for <a href="AI_in_China" class="mw-redirect" title="AI in China">AI in China</a>. According to the guidelines, researchers must ensure that AI abides by shared human values, is always under human control, and does not endanger public safety.<a href="#cite_note-188">[180]</a>

Also in September 2021, the <a href="UK" class="mw-redirect" title="UK">UK</a> published its 10-year National AI Strategy,<a href="#cite_note-189">[181]</a> which says the British government "takes the long term risk of non-aligned Artificial General Intelligence, and the unforeseeable changes that it would mean for ... the world, seriously".<a href="#cite_note-190">[182]</a> The strategy describes actions to assess long-term AI risks, including catastrophic risks.<a href="#cite_note-191">[183]</a>

In March 2021, the US National Security Commission on Artificial Intelligence said: "Advances in AI ... could lead to inflection points or leaps in capabilities. Such advances may also introduce new concerns and risks and the need for new policies, recommendations, and technical advances to ensure that systems are aligned with goals and values, including safety, robustness, and trustworthiness. The US should ... ensure that AI systems and their uses align with our goals and values."<a href="#cite_note-192">[184]</a>

In the European Union, AIs must align with <a href="Substantive_equality" title="Substantive equality">substantive equality</a> to comply with EU <a href="Non-discrimination_law" class="mw-redirect" title="Non-discrimination law">non-discrimination law</a><a href="#cite_note-193">[185]</a> and the <a href="Court_of_Justice_of_the_European_Union" title="Court of Justice of the European Union">Court of Justice of the European Union</a>.<a href="#cite_note-194">[186]</a> But the EU has yet to specify with technical rigor how it would evaluate whether AIs are aligned or in compliance.

<div class="mw-heading mw-heading2"><h2 id="Dynamic_nature_of_alignment">Dynamic nature of alignment</h2></div>

AI alignment is often perceived as a fixed objective, but some researchers argue it would be more appropriate to view alignment as an evolving process.<a href="#cite_note-195">[187]</a> One view is that AI technologies advance and human values and preferences change, alignment solutions must also adapt dynamically.<a href="#cite_note-:4-35">[35]</a> Another is that alignment solutions need not adapt if researchers can create intent-aligned AI: AI that changes its behavior automatically as human intent changes.<a href="#cite_note-196">[188]</a> The first view would have several implications:

<ul><li>AI alignment solutions require continuous updating in response to AI advancements. A static, one-time alignment approach may not suffice.<a href="#cite_note-197">[189]</a></li></ul>

<ul><li>Varying historical contexts and technological landscapes may necessitate distinct alignment strategies. This calls for a flexible approach and responsiveness to changing conditions.<a href="#cite_note-198">[190]</a></li></ul>

<ul><li>The feasibility of a permanent, "fixed" alignment solution remains uncertain. This raises the potential need for continuous oversight of the AI-human relationship.<a href="#cite_note-199">[191]</a></li></ul>

<ul><li>AI developers may have to continuously refine their ethical frameworks to ensure that their systems align with evolving human values.<a href="#cite_note-:4-35">[35]</a></li></ul>

In essence, AI alignment may not be a static destination but rather an open, flexible process. Alignment solutions that continually adapt to ethical considerations may offer the most robust approach.<a href="#cite_note-:4-35">[35]</a> This perspective could guide both effective policy-making and technical research in AI.

/* start https://en.wikipedia.org/ */

.mw-parser-output .div-col{margin-top:0.3em;column-width:30em}.mw-parser-output .div-col-small{font-size:90%}.mw-parser-output .div-col-rules{column-rule:1px solid #aaa}.mw-parser-output .div-col dl,.mw-parser-output .div-col ol,.mw-parser-output .div-col ul{margin-top:0}.mw-parser-output .div-col li,.mw-parser-output .div-col dd{page-break-inside:avoid;break-inside:avoid-column}

/* end https://en.wikipedia.org/ */

</style><div class="div-col" style="column-width: 20em;">

<ul><li><a href="AI_safety" title="AI safety">AI safety</a></li>

<li><a href="Artificial_intelligence_detection_software" class="mw-redirect" title="Artificial intelligence detection software">Artificial intelligence detection software</a></li>

<li><a href="Artificial_intelligence_and_elections" title="Artificial intelligence and elections">Artificial intelligence and elections</a></li>

<li><a href="Statement_on_AI_risk_of_extinction" class="mw-redirect" title="Statement on AI risk of extinction">Statement on AI risk of extinction</a></li>

<li><a href="Existential_risk_from_artificial_general_intelligence" class="mw-redirect" title="Existential risk from artificial general intelligence">Existential risk from artificial general intelligence</a></li>

<li><a href="AI_takeover" title="AI takeover">AI takeover</a></li>

<li><a href="AI_capability_control" title="AI capability control">AI capability control</a></li>

<li><a href="Reinforcement_learning_from_human_feedback" title="Reinforcement learning from human feedback">Reinforcement learning from human feedback</a></li>

<li><a href="Regulation_of_artificial_intelligence" title="Regulation of artificial intelligence">Regulation of artificial intelligence</a></li>

<li><a href="Artificial_wisdom" title="Artificial wisdom">Artificial wisdom</a></li>

<li><a href="Multivac" title="Multivac">Multivac</a></li>

<li><a href="Open_Letter_on_Artificial_Intelligence" class="mw-redirect" title="Open Letter on Artificial Intelligence">Open Letter on Artificial Intelligence</a></li>

<li><a href="Three_Laws_of_Robotics" title="Three Laws of Robotics">Three Laws of Robotics</a></li>

<li><a href="Toronto_Declaration" title="Toronto Declaration">Toronto Declaration</a></li>

<li><a href="Asilomar_Conference_on_Beneficial_AI" title="Asilomar Conference on Beneficial AI">Asilomar Conference on Beneficial AI</a></li>

<li><a href="Socialization" title="Socialization">Socialization</a></li></ul>

</div>

<div class="mw-heading mw-heading2"><h2 id="Footnotes">Footnotes</h2></div>

/* start https://en.wikipedia.org/ */

.mw-parser-output .reflist{margin-bottom:0.5em;list-style-type:decimal}@media screen{.mw-parser-output .reflist{font-size:90%}}.mw-parser-output .reflist .references{font-size:100%;margin-bottom:0;list-style-type:inherit}.mw-parser-output .reflist-columns-2{column-width:30em}.mw-parser-output .reflist-columns-3{column-width:25em}.mw-parser-output .reflist-columns{margin-top:0.3em}.mw-parser-output .reflist-columns ol{margin-top:0}.mw-parser-output .reflist-columns li{page-break-inside:avoid;break-inside:avoid-column}.mw-parser-output .reflist-upper-alpha{list-style-type:upper-alpha}.mw-parser-output .reflist-upper-roman{list-style-type:upper-roman}.mw-parser-output .reflist-lower-alpha{list-style-type:lower-alpha}.mw-parser-output .reflist-lower-greek{list-style-type:lower-greek}.mw-parser-output .reflist-lower-roman{list-style-type:lower-roman}

/* end https://en.wikipedia.org/ */

</style><div class="reflist reflist-lower-alpha">

<li id="cite_note-37"><a href="#cite_ref-37">^</a> Terminology varies based on context. Similar concepts include goal function, utility function, loss function, etc.

</li>

<li id="cite_note-38"><a href="#cite_ref-38">^</a> or minimize, depending on the context

</li>

<li id="cite_note-39"><a href="#cite_ref-39">^</a> in the presence of uncertainty, the <a href="Expected_value" title="Expected value">expected value</a>

</li>

<li id="cite_note-90"><a href="#cite_ref-90">^</a> In a 1951 lecture<a href="#cite_note-88">[85]</a> Turing argued that "It seems probable that once the machine thinking method had started, it would not take long to outstrip our feeble powers. There would be no question of the machines dying, and they would be able to converse with each other to sharpen their wits. At some stage therefore we should have to expect the machines to take control, in the way that is mentioned in Samuel Butler's Erewhon." Also in a lecture broadcast on BBC<a href="#cite_note-89">[86]</a> expressed: "If a machine can think, it might think more intelligently than we do, and then where should we be? Even if we could keep the machines in a subservient position, for instance by turning off the power at strategic moments, we should, as a species, feel greatly humbled.... This new danger... is certainly something which can give us anxiety."

</li>

<li id="cite_note-92"><a href="#cite_ref-92">^</a> Pearl wrote "Human Compatible made me a convert to Russell's concerns with our ability to control our upcoming creation–super-intelligent machines. Unlike outside alarmists and futurists, Russell is a leading authority on AI. His new book will educate the public about AI more than any book I can think of, and is a delightful and uplifting read" about Russell's book Human Compatible: AI and the Problem of Control<a href="#cite_note-:2102-5">[5]</a> which argues that existential risk to humanity from misaligned AI is a serious concern worth addressing today.

</li>

<li id="cite_note-94"><a href="#cite_ref-94">^</a> Russell & Norvig<a href="#cite_note-AIMA-16">[16]</a> note: "The "King Midas problem" was anticipated by Marvin Minsky, who once suggested that an AI program designed to solve the Riemann Hypothesis might end up taking over all the resources of Earth to build more powerful supercomputers."

</li>

<li id="cite_note-119"><a href="#cite_ref-119">^</a> Vincent Wiegel argued "we should extend [machines] with moral sensitivity to the moral dimensions of the situations in which the increasingly autonomous machines will inevitably find themselves.",<a href="#cite_note-117">[111]</a> referencing the book Moral machines: teaching robots right from wrong<a href="#cite_note-118">[112]</a> from Wendell Wallach and Colin Allen.

</li>

<li id="cite_note-157"><a href="#cite_ref-157">^</a> On the one hand, currently popular systems such as chatbots only provide services of limited scope lasting no longer than the time of a conversation, which requires little or no planning. The success of such approaches may indicate that future systems will also lack goal-directed planning, especially over long horizons. On the other hand, models are increasingly trained using goal-directed methods such as reinforcement learning (e.g. ChatGPT) and explicitly planning architectures (e.g. AlphaGo Zero). As planning over long horizons is often helpful for humans, some researchers argue that companies will automate it once models become capable of it.<a href="#cite_note-Carlsmith2022-4">[4]</a> Similarly, political leaders may see an advance in developing powerful AI systems that can outmaneuver adversaries through planning. Alternatively, long-term planning might emerge as a byproduct because it is useful e.g. for models that are trained to predict the actions of humans who themselves perform long-term planning.<a href="#cite_note-Opportunities_Risks-10">[10]</a> Nonetheless, the majority of AI systems may remain myopic and perform no long-term planning.

</li>

</ol></div></div>

<div class="mw-heading mw-heading2"><h2 id="References">References</h2></div>

/* start https://en.wikipedia.org/ */

.mw-parser-output cite.citation{font-style:inherit;word-wrap:break-word}.mw-parser-output .citation q{quotes:"\"""\"""'""'"}.mw-parser-output .citation:target{background-color:rgba(0,127,255,0.133)}.mw-parser-output .id-lock-free.id-lock-free a{background:url("./mw/Lock-green.svg")right 0.1em center/9px no-repeat}.mw-parser-output .id-lock-limited.id-lock-limited a,.mw-parser-output .id-lock-registration.id-lock-registration a{background:url("./mw/Lock-gray-alt-2.svg")right 0.1em center/9px no-repeat}.mw-parser-output .id-lock-subscription.id-lock-subscription a{background:url("./mw/Lock-red-alt-2.svg")right 0.1em center/9px no-repeat}.mw-parser-output .cs1-ws-icon a{background:url("./mw/Wikisource-logo.svg")right 0.1em center/12px no-repeat}body:not(.skin-timeless):not(.skin-minerva) .mw-parser-output .id-lock-free a,body:not(.skin-timeless):not(.skin-minerva) .mw-parser-output .id-lock-limited a,body:not(.skin-timeless):not(.skin-minerva) .mw-parser-output .id-lock-registration a,body:not(.skin-timeless):not(.skin-minerva) .mw-parser-output .id-lock-subscription a,body:not(.skin-timeless):not(.skin-minerva) .mw-parser-output .cs1-ws-icon a{background-size:contain;padding:0 1em 0 0}.mw-parser-output .cs1-code{color:inherit;background:inherit;border:none;padding:inherit}.mw-parser-output .cs1-hidden-error{display:none;color:var(--color-error,#d33)}.mw-parser-output .cs1-visible-error{color:var(--color-error,#d33)}.mw-parser-output .cs1-maint{display:none;color:#085;margin-left:0.3em}.mw-parser-output .cs1-kern-left{padding-left:0.2em}.mw-parser-output .cs1-kern-right{padding-right:0.2em}.mw-parser-output .citation .mw-selflink{font-weight:inherit}@media screen{.mw-parser-output .cs1-format{font-size:95%}html.skin-theme-clientpref-night .mw-parser-output .cs1-maint{color:#18911f}}@media screen and (prefers-color-scheme:dark){html.skin-theme-clientpref-os .mw-parser-output .cs1-maint{color:#18911f}}

/* end https://en.wikipedia.org/ */

</style><cite id="CITEREFRussellNorvig2021" class="citation book cs1">Russell, Stuart J.; Norvig, Peter (2021). <a rel="nofollow" class="external text" href="https://www.pearson.com/us/higher-education/program/Russell-Artificial-Intelligence-A-Modern-Approach-4th-Edition/PGM1263338.html">Artificial intelligence: A modern approach</a> (4th ed.). Pearson. pp. 5, 1003. <a href="ISBN_(identifier)" class="mw-redirect" title="ISBN (identifier)">ISBN</a> <bdi>9780134610993</bdi>. Retrieved September 12, 2022.</cite>

</li>

<li id="cite_note-dlp2023-2">^ <a href="#cite_ref-dlp2023_2-0">a</a> <a href="#cite_ref-dlp2023_2-1">b</a> <a href="#cite_ref-dlp2023_2-2">c</a> <a href="#cite_ref-dlp2023_2-3">d</a> <cite id="CITEREFNgoChanMindermann2022" class="citation journal cs1">Ngo, Richard; Chan, Lawrence; Mindermann, Sören (2022). "The Alignment Problem from a Deep Learning Perspective". International Conference on Learning Representations. <a href="ArXiv_(identifier)" class="mw-redirect" title="ArXiv (identifier)">arXiv</a>:<a rel="nofollow" class="external text" href="https://arxiv.org/abs/2209.00626">2209.00626</a>.</cite>

</li>

<li id="cite_note-mmmm2022-3">^ <a href="#cite_ref-mmmm2022_3-0">a</a> <a href="#cite_ref-mmmm2022_3-1">b</a> <a href="#cite_ref-mmmm2022_3-2">c</a> <a href="#cite_ref-mmmm2022_3-3">d</a> <a href="#cite_ref-mmmm2022_3-4">e</a> <a href="#cite_ref-mmmm2022_3-5">f</a> <cite id="CITEREFPanBhatiaSteinhardt2022" class="citation conference cs1">Pan, Alexander; Bhatia, Kush; Steinhardt, Jacob (February 14, 2022). <a rel="nofollow" class="external text" href="https://openreview.net/forum?id=JYtwGwIL7ye">The Effects of Reward Misspecification: Mapping and Mitigating Misaligned Models</a>. International Conference on Learning Representations. Retrieved July 21, 2022.</cite>

</li>

<li id="cite_note-Carlsmith2022-4">^ <a href="#cite_ref-Carlsmith2022_4-0">a</a> <a href="#cite_ref-Carlsmith2022_4-1">b</a> <a href="#cite_ref-Carlsmith2022_4-2">c</a> <a href="#cite_ref-Carlsmith2022_4-3">d</a> <a href="#cite_ref-Carlsmith2022_4-4">e</a> <a href="#cite_ref-Carlsmith2022_4-5">f</a> <a href="#cite_ref-Carlsmith2022_4-6">g</a> <a href="#cite_ref-Carlsmith2022_4-7">h</a> <a href="#cite_ref-Carlsmith2022_4-8">i</a> <a href="#cite_ref-Carlsmith2022_4-9">j</a> <a href="#cite_ref-Carlsmith2022_4-10">k</a> <a href="#cite_ref-Carlsmith2022_4-11">l</a> <cite id="CITEREFCarlsmith2022" class="citation arxiv cs1">Carlsmith, Joseph (June 16, 2022). "Is Power-Seeking AI an Existential Risk?". <a href="ArXiv_(identifier)" class="mw-redirect" title="ArXiv (identifier)">arXiv</a>:<a rel="nofollow" class="external text" href="https://arxiv.org/abs/2206.13353">2206.13353</a> [<a rel="nofollow" class="external text" href="https://arxiv.org/archive/cs.CY">cs.CY</a>].</cite>

</li>

<li id="cite_note-:2102-5">^ <a href="#cite_ref-:2102_5-0">a</a> <a href="#cite_ref-:2102_5-1">b</a> <a href="#cite_ref-:2102_5-2">c</a> <a href="#cite_ref-:2102_5-3">d</a> <a href="#cite_ref-:2102_5-4">e</a> <a href="#cite_ref-:2102_5-5">f</a> <a href="#cite_ref-:2102_5-6">g</a> <a href="#cite_ref-:2102_5-7">h</a> <a href="#cite_ref-:2102_5-8">i</a> <a href="#cite_ref-:2102_5-9">j</a> <a href="#cite_ref-:2102_5-10">k</a> <a href="#cite_ref-:2102_5-11">l</a> <a href="#cite_ref-:2102_5-12">m</a> <a href="#cite_ref-:2102_5-13">n</a> <a href="#cite_ref-:2102_5-14">o</a> <a href="#cite_ref-:2102_5-15">p</a> <a href="#cite_ref-:2102_5-16">q</a> <cite id="CITEREFRussell2020" class="citation book cs1">Russell, Stuart J. (2020). <a rel="nofollow" class="external text" href="https://www.penguinrandomhouse.com/books/566677/human-compatible-by-stuart-russell/">Human compatible: Artificial intelligence and the problem of control</a>. Penguin Random House. <a href="ISBN_(identifier)" class="mw-redirect" title="ISBN (identifier)">ISBN</a> <bdi>9780525558637</bdi>. <a href="OCLC_(identifier)" class="mw-redirect" title="OCLC (identifier)">OCLC</a> <a rel="nofollow" class="external text" href="https://search.worldcat.org/oclc/1113410915">1113410915</a>.</cite>

</li>

<li id="cite_note-Christian2020-6">^ <a href="#cite_ref-Christian2020_6-0">a</a> <a href="#cite_ref-Christian2020_6-1">b</a> <a href="#cite_ref-Christian2020_6-2">c</a> <a href="#cite_ref-Christian2020_6-3">d</a> <a href="#cite_ref-Christian2020_6-4">e</a> <a href="#cite_ref-Christian2020_6-5">f</a> <cite id="CITEREFChristian2020" class="citation book cs1">Christian, Brian (2020). <a rel="nofollow" class="external text" href="https://wwnorton.co.uk/books/9780393635829-the-alignment-problem">The alignment problem: Machine learning and human values</a>. W. W. Norton & Company. <a href="ISBN_(identifier)" class="mw-redirect" title="ISBN (identifier)">ISBN</a> <bdi>978-0-393-86833-3</bdi>. <a href="OCLC_(identifier)" class="mw-redirect" title="OCLC (identifier)">OCLC</a> <a rel="nofollow" class="external text" href="https://search.worldcat.org/oclc/1233266753">1233266753</a>. <a rel="nofollow" class="external text" href="https://web.archive.org/web/20230210114137/https://wwnorton.co.uk/books/9780393635829-the-alignment-problem">Archived</a> from the original on February 10, 2023. Retrieved September 12, 2022.</cite>

</li>

<li id="cite_note-gmdrl-7">^ <a href="#cite_ref-gmdrl_7-0">a</a> <a href="#cite_ref-gmdrl_7-1">b</a> <a href="#cite_ref-gmdrl_7-2">c</a> <a href="#cite_ref-gmdrl_7-3">d</a> <cite id="CITEREFLangoscoKochSharkeyPfau2022" class="citation conference cs1">Langosco, Lauro Langosco Di; Koch, Jack; Sharkey, Lee D.; Pfau, Jacob; Krueger, David (June 28, 2022). <a rel="nofollow" class="external text" href="https://proceedings.mlr.press/v162/langosco22a.html">"Goal Misgeneralization in Deep Reinforcement Learning"</a>. Proceedings of the 39th International Conference on Machine Learning. International Conference on Machine Learning. PMLR. pp. 12004–12019. Retrieved March 11, 2023.</cite>

</li>

<li id="cite_note-8"><a href="#cite_ref-8">^</a> <cite id="CITEREFPillay2024" class="citation magazine cs1">Pillay, Tharin (December 15, 2024). <a rel="nofollow" class="external text" href="https://time.com/7202312/new-tests-reveal-ai-capacity-for-deception/">"New Tests Reveal AI's Capacity for Deception"</a>. TIME. Retrieved January 12, 2025.</cite>

</li>

<li id="cite_note-9"><a href="#cite_ref-9">^</a> <cite id="CITEREFPerrigo2024" class="citation magazine cs1">Perrigo, Billy (December 18, 2024). <a rel="nofollow" class="external text" href="https://time.com/7202784/ai-research-strategic-lying/">"Exclusive: New Research Shows AI Strategically Lying"</a>. TIME. Retrieved January 12, 2025.</cite>

</li>

<li id="cite_note-Opportunities_Risks-10">^ <a href="#cite_ref-Opportunities_Risks_10-0">a</a> <a href="#cite_ref-Opportunities_Risks_10-1">b</a> <a href="#cite_ref-Opportunities_Risks_10-2">c</a> <a href="#cite_ref-Opportunities_Risks_10-3">d</a> <a href="#cite_ref-Opportunities_Risks_10-4">e</a> <a href="#cite_ref-Opportunities_Risks_10-5">f</a> <a href="#cite_ref-Opportunities_Risks_10-6">g</a> <a href="#cite_ref-Opportunities_Risks_10-7">h</a> <a href="#cite_ref-Opportunities_Risks_10-8">i</a> <cite id="CITEREFBommasaniHudsonAdeliAltman2022" class="citation journal cs1">Bommasani, Rishi; Hudson, Drew A.; Adeli, Ehsan; Altman, Russ; Arora, Simran; von Arx, Sydney; Bernstein, Michael S.; Bohg, Jeannette; Bosselut, Antoine; Brunskill, Emma; Brynjolfsson, Erik (July 12, 2022). <a rel="nofollow" class="external text" href="https://fsi.stanford.edu/publication/opportunities-and-risks-foundation-models">"On the Opportunities and Risks of Foundation Models"</a>. Stanford CRFM. <a href="ArXiv_(identifier)" class="mw-redirect" title="ArXiv (identifier)">arXiv</a>:<a rel="nofollow" class="external text" href="https://arxiv.org/abs/2108.07258">2108.07258</a>.</cite>

</li>

<li id="cite_note-feedback2022-11">^ <a href="#cite_ref-feedback2022_11-0">a</a> <a href="#cite_ref-feedback2022_11-1">b</a> <a href="#cite_ref-feedback2022_11-2">c</a> <cite id="CITEREFOuyangWuJiangAlmeida2022" class="citation arxiv cs1">Ouyang, Long; Wu, Jeff; Jiang, Xu; Almeida, Diogo; Wainwright, Carroll L.; Mishkin, Pamela; Zhang, Chong; Agarwal, Sandhini; Slama, Katarina; Ray, Alex; Schulman, J.; Hilton, Jacob; Kelton, Fraser; Miller, Luke E.; Simens, Maddie; Askell, Amanda; Welinder, P.; Christiano, P.; Leike, J.; Lowe, Ryan J. (2022). "Training language models to follow instructions with human feedback". <a href="ArXiv_(identifier)" class="mw-redirect" title="ArXiv (identifier)">arXiv</a>:<a rel="nofollow" class="external text" href="https://arxiv.org/abs/2203.02155">2203.02155</a> [<a rel="nofollow" class="external text" href="https://arxiv.org/archive/cs.CL">cs.CL</a>].</cite>

</li>

<li id="cite_note-OpenAICodex-12">^ <a href="#cite_ref-OpenAICodex_12-0">a</a> <a href="#cite_ref-OpenAICodex_12-1">b</a> <cite id="CITEREFZarembaBrockmanOpenAI2021" class="citation web cs1">Zaremba, Wojciech; Brockman, Greg; OpenAI (August 10, 2021). <a rel="nofollow" class="external text" href="https://openai.com/blog/openai-codex/">"OpenAI Codex"</a>. OpenAI. <a rel="nofollow" class="external text" href="https://web.archive.org/web/20230203201912/https://openai.com/blog/openai-codex/">Archived</a> from the original on February 3, 2023. Retrieved July 23, 2022.</cite>

</li>

<li id="cite_note-13"><a href="#cite_ref-13">^</a> <cite id="CITEREFKoberBagnellPeters2013" class="citation journal cs1">Kober, Jens; Bagnell, J. Andrew; Peters, Jan (September 1, 2013). <a rel="nofollow" class="external text" href="http://journals.sagepub.com/doi/10.1177/0278364913495721">"Reinforcement learning in robotics: A survey"</a>. The International Journal of Robotics Research. 32 (11): 1238–1274. <a href="Doi_(identifier)" class="mw-redirect" title="Doi (identifier)">doi</a>:<a rel="nofollow" class="external text" href="https://doi.org/10.1177%2F0278364913495721">10.1177/0278364913495721</a>. <a href="ISSN_(identifier)" class="mw-redirect" title="ISSN (identifier)">ISSN</a> <a rel="nofollow" class="external text" href="https://search.worldcat.org/issn/0278-3649">0278-3649</a>. <a href="S2CID_(identifier)" class="mw-redirect" title="S2CID (identifier)">S2CID</a> <a rel="nofollow" class="external text" href="https://api.semanticscholar.org/CorpusID:1932843">1932843</a>. <a rel="nofollow" class="external text" href="https://web.archive.org/web/20221015200445/https://journals.sagepub.com/doi/10.1177/0278364913495721">Archived</a> from the original on October 15, 2022. Retrieved September 12, 2022.</cite>

</li>

<li id="cite_note-14"><a href="#cite_ref-14">^</a> <cite id="CITEREFKnoxAllieviBanzhafSchmitt2023" class="citation journal cs1">Knox, W. Bradley; Allievi, Alessandro; Banzhaf, Holger; Schmitt, Felix; Stone, Peter (March 1, 2023). <a rel="nofollow" class="external text" href="https://doi.org/10.1016%2Fj.artint.2022.103829">"Reward (Mis)design for autonomous driving"</a>. Artificial Intelligence. 316 103829. <a href="ArXiv_(identifier)" class="mw-redirect" title="ArXiv (identifier)">arXiv</a>:<a rel="nofollow" class="external text" href="https://arxiv.org/abs/2104.13906">2104.13906</a>. <a href="Doi_(identifier)" class="mw-redirect" title="Doi (identifier)">doi</a>:<a rel="nofollow" class="external text" href="https://doi.org/10.1016%2Fj.artint.2022.103829">10.1016/j.artint.2022.103829</a>. <a href="ISSN_(identifier)" class="mw-redirect" title="ISSN (identifier)">ISSN</a> <a rel="nofollow" class="external text" href="https://search.worldcat.org/issn/0004-3702">0004-3702</a>. <a href="S2CID_(identifier)" class="mw-redirect" title="S2CID (identifier)">S2CID</a> <a rel="nofollow" class="external text" href="https://api.semanticscholar.org/CorpusID:233423198">233423198</a>.</cite>

</li>

<li id="cite_note-15"><a href="#cite_ref-15">^</a> <cite id="CITEREFStray2020" class="citation journal cs1">Stray, Jonathan (2020). <a rel="nofollow" class="external text" href="https://www.ncbi.nlm.nih.gov/pmc/articles/PMC7610010">"Aligning AI Optimization to Community Well-Being"</a>. International Journal of Community Well-Being. 3 (4): 443–463. <a href="Doi_(identifier)" class="mw-redirect" title="Doi (identifier)">doi</a>:<a rel="nofollow" class="external text" href="https://doi.org/10.1007%2Fs42413-020-00086-3">10.1007/s42413-020-00086-3</a>. <a href="ISSN_(identifier)" class="mw-redirect" title="ISSN (identifier)">ISSN</a> <a rel="nofollow" class="external text" href="https://search.worldcat.org/issn/2524-5295">2524-5295</a>. <a href="PMC_(identifier)" class="mw-redirect" title="PMC (identifier)">PMC</a> <a rel="nofollow" class="external text" href="https://www.ncbi.nlm.nih.gov/pmc/articles/PMC7610010">7610010</a>. <a href="PMID_(identifier)" class="mw-redirect" title="PMID (identifier)">PMID</a> <a rel="nofollow" class="external text" href="https://pubmed.ncbi.nlm.nih.gov/34723107">34723107</a>. <a href="S2CID_(identifier)" class="mw-redirect" title="S2CID (identifier)">S2CID</a> <a rel="nofollow" class="external text" href="https://api.semanticscholar.org/CorpusID:226254676">226254676</a>.</cite>

</li>

<li id="cite_note-AIMA-16">^ <a href="#cite_ref-AIMA_16-0">a</a> <a href="#cite_ref-AIMA_16-1">b</a> <cite id="CITEREFRussellNorvig2009" class="citation book cs1">Russell, Stuart; Norvig, Peter (2009). <a rel="nofollow" class="external text" href="https://aima.cs.berkeley.edu/">Artificial Intelligence: A Modern Approach</a>. Prentice Hall. p. 1003. <a href="ISBN_(identifier)" class="mw-redirect" title="ISBN (identifier)">ISBN</a> <bdi>978-0-13-461099-3</bdi>.</cite>

</li>

<li id="cite_note-:2-17">^ <a href="#cite_ref-:2_17-0">a</a> <a href="#cite_ref-:2_17-1">b</a> <cite id="CITEREFSmith" class="citation web cs1">Smith, Craig S. <a rel="nofollow" class="external text" href="https://www.forbes.com/sites/craigsmith/2023/05/04/geoff-hinton-ais-most-famous-researcher-warns-of-existential-threat/">"Geoff Hinton, AI's Most Famous Researcher, Warns Of 'Existential Threat'"</a>. Forbes. Retrieved May 4, 2023.</cite>

</li>

<li id="cite_note-18"><a href="#cite_ref-18">^</a> <cite id="CITEREFBengioHintonYaoSong2024" class="citation journal cs1">Bengio, Yoshua; Hinton, Geoffrey; Yao, Andrew; Song, Dawn; Abbeel, Pieter; Harari, Yuval Noah; Zhang, Ya-Qin; Xue, Lan; Shalev-Shwartz, Shai (2024). "Managing extreme AI risks amid rapid progress". Science. 384 (6698): 842–845. <a href="ArXiv_(identifier)" class="mw-redirect" title="ArXiv (identifier)">arXiv</a>:<a rel="nofollow" class="external text" href="https://arxiv.org/abs/2310.17688">2310.17688</a>. <a href="Bibcode_(identifier)" class="mw-redirect" title="Bibcode (identifier)">Bibcode</a>:<a rel="nofollow" class="external text" href="https://ui.adsabs.harvard.edu/abs/2024Sci...384..842B">2024Sci...384..842B</a>. <a href="Doi_(identifier)" class="mw-redirect" title="Doi (identifier)">doi</a>:<a rel="nofollow" class="external text" href="https://doi.org/10.1126%2Fscience.adn0117">10.1126/science.adn0117</a>. <a href="PMID_(identifier)" class="mw-redirect" title="PMID (identifier)">PMID</a> <a rel="nofollow" class="external text" href="https://pubmed.ncbi.nlm.nih.gov/38768279">38768279</a>.</cite>

</li>

<li id="cite_note-19"><a href="#cite_ref-19">^</a> <cite class="citation web cs1"><a rel="nofollow" class="external text" href="https://www.safe.ai/statement-on-ai-risk">"Statement on AI Risk | CAIS"</a>. www.safe.ai. Retrieved February 11, 2024.</cite>

</li>

<li id="cite_note-20"><a href="#cite_ref-20">^</a> <cite id="CITEREFGraceStewartSandkühlerThomas2024" class="citation arxiv cs1">Grace, Katja; Stewart, Harlan; Sandkühler, Julia Fabienne; Thomas, Stephen; Weinstein-Raun, Ben; Brauner, Jan (January 5, 2024). "Thousands of AI Authors on the Future of AI". <a href="ArXiv_(identifier)" class="mw-redirect" title="ArXiv (identifier)">arXiv</a>:<a rel="nofollow" class="external text" href="https://arxiv.org/abs/2401.02843">2401.02843</a> [<a rel="nofollow" class="external text" href="https://arxiv.org/archive/cs.CY">cs.CY</a>].</cite>

</li>

<li id="cite_note-21"><a href="#cite_ref-21">^</a> <cite id="CITEREFPerrigo2024" class="citation magazine cs1">Perrigo, Billy (February 13, 2024). <a rel="nofollow" class="external text" href="https://time.com/6694432/yann-lecun-meta-ai-interview/">"Meta's AI Chief Yann LeCun on AGI, Open-Source, and AI Risk"</a>. TIME. Retrieved June 26, 2024.</cite>

</li>

<li id="cite_note-22"><a href="#cite_ref-22">^</a> <cite class="citation web cs1"><a rel="nofollow" class="external text" href="https://www.techtarget.com/whatis/definition/AI-alignment">"What is AI alignment?"</a>. <a href="TechTarget" class="mw-redirect" title="TechTarget">TechTarget</a>. May 3, 2023. Retrieved June 28, 2025.</cite>

</li>

<li id="cite_note-23"><a href="#cite_ref-23">^</a> <cite id="CITEREFAhmedJaźwińskaAhlawatWinecoff2024" class="citation journal cs1">Ahmed, Shazeda; Jaźwińska, Klaudia; Ahlawat, Archana; Winecoff, Amy; Wang, Mona (April 14, 2024). <a rel="nofollow" class="external text" href="https://firstmonday.org/ojs/index.php/fm/article/view/13626">"Field-building and the epistemic culture of AI safety"</a>. First Monday. <a href="Doi_(identifier)" class="mw-redirect" title="Doi (identifier)">doi</a>:<a rel="nofollow" class="external text" href="https://doi.org/10.5210%2Ffm.v29i4.13626">10.5210/fm.v29i4.13626</a>. <a href="ISSN_(identifier)" class="mw-redirect" title="ISSN (identifier)">ISSN</a> <a rel="nofollow" class="external text" href="https://search.worldcat.org/issn/1396-0466">1396-0466</a>.</cite>

</li>

<li id="cite_note-building2018-24">^ <a href="#cite_ref-building2018_24-0">a</a> <a href="#cite_ref-building2018_24-1">b</a> <a href="#cite_ref-building2018_24-2">c</a> <a href="#cite_ref-building2018_24-3">d</a> <cite id="CITEREFOrtegaMainiDeepMind_safety_team2018" class="citation web cs1">Ortega, Pedro A.; Maini, Vishal; DeepMind safety team (September 27, 2018). <a rel="nofollow" class="external text" href="https://deepmindsafetyresearch.medium.com/building-safe-artificial-intelligence-52f5f75058f1">"Building safe artificial intelligence: specification, robustness, and assurance"</a>. DeepMind Safety Research – Medium. <a rel="nofollow" class="external text" href="https://web.archive.org/web/20230210114142/https://deepmindsafetyresearch.medium.com/building-safe-artificial-intelligence-52f5f75058f1">Archived</a> from the original on February 10, 2023. Retrieved July 18, 2022.</cite>

</li>

<li id="cite_note-:333-25">^ <a href="#cite_ref-:333_25-0">a</a> <a href="#cite_ref-:333_25-1">b</a> <cite id="CITEREFRorvig2022" class="citation web cs1">Rorvig, Mordechai (April 14, 2022). <a rel="nofollow" class="external text" href="https://www.quantamagazine.org/researchers-glimpse-how-ai-gets-so-good-at-language-processing-20220414/">"Researchers Gain New Understanding From Simple AI"</a>. Quanta Magazine. <a rel="nofollow" class="external text" href="https://web.archive.org/web/20230210114056/https://www.quantamagazine.org/researchers-glimpse-how-ai-gets-so-good-at-language-processing-20220414/">Archived</a> from the original on February 10, 2023. Retrieved July 18, 2022.</cite>

</li>

<li id="cite_note-26"><a href="#cite_ref-26">^</a> <cite id="CITEREFDoshi-VelezKim2017" class="citation arxiv cs1">Doshi-Velez, Finale; Kim, Been (March 2, 2017). "Towards A Rigorous Science of Interpretable Machine Learning". <a href="ArXiv_(identifier)" class="mw-redirect" title="ArXiv (identifier)">arXiv</a>:<a rel="nofollow" class="external text" href="https://arxiv.org/abs/1702.08608">1702.08608</a> [<a rel="nofollow" class="external text" href="https://arxiv.org/archive/stat.ML">stat.ML</a>].</cite>

<ul><li><cite id="CITEREFWiblin2021" class="citation podcast cs1">Wiblin, Robert (August 4, 2021). <a rel="nofollow" class="external text" href="https://80000hours.org/podcast/episodes/chris-olah-interpretability-research/">"Chris Olah on what the hell is going on inside neural networks"</a> (Podcast). 80,000 hours. No. 107. Retrieved July 23, 2022.</cite></li></ul>

</li>

<li id="cite_note-concrete2016-27">^ <a href="#cite_ref-concrete2016_27-0">a</a> <a href="#cite_ref-concrete2016_27-1">b</a> <a href="#cite_ref-concrete2016_27-2">c</a> <a href="#cite_ref-concrete2016_27-3">d</a> <a href="#cite_ref-concrete2016_27-4">e</a> <a href="#cite_ref-concrete2016_27-5">f</a> <a href="#cite_ref-concrete2016_27-6">g</a> <a href="#cite_ref-concrete2016_27-7">h</a> <a href="#cite_ref-concrete2016_27-8">i</a> <a href="#cite_ref-concrete2016_27-9">j</a> <a href="#cite_ref-concrete2016_27-10">k</a> <cite id="CITEREFAmodeiOlahSteinhardtChristiano2016" class="citation arxiv cs1">Amodei, Dario; Olah, Chris; Steinhardt, Jacob; Christiano, Paul; Schulman, John; Mané, Dan (June 21, 2016). "Concrete Problems in AI Safety". <a href="ArXiv_(identifier)" class="mw-redirect" title="ArXiv (identifier)">arXiv</a>:<a rel="nofollow" class="external text" href="https://arxiv.org/abs/1606.06565">1606.06565</a> [<a rel="nofollow" class="external text" href="https://arxiv.org/archive/cs.AI">cs.AI</a>].</cite>

</li>

<li id="cite_note-28"><a href="#cite_ref-28">^</a> <cite id="CITEREFRussellDeweyTegmark2015" class="citation journal cs1">Russell, Stuart; Dewey, Daniel; Tegmark, Max (December 31, 2015). <a rel="nofollow" class="external text" href="https://ojs.aaai.org/index.php/aimagazine/article/view/2577">"Research Priorities for Robust and Beneficial Artificial Intelligence"</a>. AI Magazine. 36 (4): 105–114. <a href="ArXiv_(identifier)" class="mw-redirect" title="ArXiv (identifier)">arXiv</a>:<a rel="nofollow" class="external text" href="https://arxiv.org/abs/1602.03506">1602.03506</a>. <a href="Doi_(identifier)" class="mw-redirect" title="Doi (identifier)">doi</a>:<a rel="nofollow" class="external text" href="https://doi.org/10.1609%2Faimag.v36i4.2577">10.1609/aimag.v36i4.2577</a>. <a href="Hdl_(identifier)" class="mw-redirect" title="Hdl (identifier)">hdl</a>:<a rel="nofollow" class="external text" href="https://hdl.handle.net/1721.1%2F108478">1721.1/108478</a>. <a href="ISSN_(identifier)" class="mw-redirect" title="ISSN (identifier)">ISSN</a> <a rel="nofollow" class="external text" href="https://search.worldcat.org/issn/2371-9621">2371-9621</a>. <a href="S2CID_(identifier)" class="mw-redirect" title="S2CID (identifier)">S2CID</a> <a rel="nofollow" class="external text" href="https://api.semanticscholar.org/CorpusID:8174496">8174496</a>. <a rel="nofollow" class="external text" href="https://web.archive.org/web/20230202181059/https://ojs.aaai.org/index.php/aimagazine/article/view/2577">Archived</a> from the original on February 2, 2023. Retrieved September 12, 2022.</cite>

</li>

<li id="cite_note-prefsurvey2017-29">^ <a href="#cite_ref-prefsurvey2017_29-0">a</a> <a href="#cite_ref-prefsurvey2017_29-1">b</a> <cite id="CITEREFWirthAkrourNeumannFürnkranz2017" class="citation journal cs1">Wirth, Christian; Akrour, Riad; Neumann, Gerhard; Fürnkranz, Johannes (2017). "A survey of preference-based reinforcement learning methods". Journal of Machine Learning Research. 18 (136): 1–46.</cite>

</li>

<li id="cite_note-drlfhp-30">^ <a href="#cite_ref-drlfhp_30-0">a</a> <a href="#cite_ref-drlfhp_30-1">b</a> <cite id="CITEREFChristianoLeikeBrownMartic2017" class="citation conference cs1">Christiano, Paul F.; Leike, Jan; Brown, Tom B.; Martic, Miljan; Legg, Shane; Amodei, Dario (2017). "Deep reinforcement learning from human preferences". Proceedings of the 31st International Conference on Neural Information Processing Systems. NIPS'17. Red Hook, NY, USA: Curran Associates Inc. pp. 4302–4310. <a href="ISBN_(identifier)" class="mw-redirect" title="ISBN (identifier)">ISBN</a> <bdi>978-1-5108-6096-4</bdi>.</cite>

</li>

<li id="cite_note-LessToxic-31">^ <a href="#cite_ref-LessToxic_31-0">a</a> <a href="#cite_ref-LessToxic_31-1">b</a> <a href="#cite_ref-LessToxic_31-2">c</a> <a href="#cite_ref-LessToxic_31-3">d</a> <a href="#cite_ref-LessToxic_31-4">e</a> <a href="#cite_ref-LessToxic_31-5">f</a> <cite id="CITEREFHeaven2022" class="citation web cs1">Heaven, Will Douglas (January 27, 2022). <a rel="nofollow" class="external text" href="https://www.technologyreview.com/2022/01/27/1044398/new-gpt3-openai-chatbot-language-model-ai-toxic-misinformation/">"The new version of GPT-3 is much better behaved (and should be less toxic)"</a>. MIT Technology Review. <a rel="nofollow" class="external text" href="https://web.archive.org/web/20230210114056/https://www.technologyreview.com/2022/01/27/1044398/new-gpt3-openai-chatbot-language-model-ai-toxic-misinformation/">Archived</a> from the original on February 10, 2023. Retrieved July 18, 2022.</cite>

</li>

<li id="cite_note-32"><a href="#cite_ref-32">^</a> <cite id="CITEREFMohseniWangYuXiao2022" class="citation arxiv cs1">Mohseni, Sina; Wang, Haotao; Yu, Zhiding; Xiao, Chaowei; Wang, Zhangyang; Yadawa, Jay (March 7, 2022). "Taxonomy of Machine Learning Safety: A Survey and Primer". <a href="ArXiv_(identifier)" class="mw-redirect" title="ArXiv (identifier)">arXiv</a>:<a rel="nofollow" class="external text" href="https://arxiv.org/abs/2106.04823">2106.04823</a> [<a rel="nofollow" class="external text" href="https://arxiv.org/archive/cs.LG">cs.LG</a>].</cite>

</li>

<li id="cite_note-33"><a href="#cite_ref-33">^</a> <cite id="CITEREFClifton2020" class="citation web cs1">Clifton, Jesse (2020). <a rel="nofollow" class="external text" href="https://longtermrisk.org/research-agenda/">"Cooperation, Conflict, and Transformative Artificial Intelligence: A Research Agenda"</a>. Center on Long-Term Risk. <a rel="nofollow" class="external text" href="https://web.archive.org/web/20230101041759/https://longtermrisk.org/research-agenda">Archived</a> from the original on January 1, 2023. Retrieved July 18, 2022.</cite>

<ul><li><cite id="CITEREFDafoeBachrachHadfieldHorvitz2021" class="citation journal cs1">Dafoe, Allan; Bachrach, Yoram; Hadfield, Gillian; Horvitz, Eric; Larson, Kate; Graepel, Thore (May 6, 2021). <a rel="nofollow" class="external text" href="http://www.nature.com/articles/d41586-021-01170-0">"Cooperative AI: machines must learn to find common ground"</a>. Nature. 593 (7857): 33–36. <a href="Bibcode_(identifier)" class="mw-redirect" title="Bibcode (identifier)">Bibcode</a>:<a rel="nofollow" class="external text" href="https://ui.adsabs.harvard.edu/abs/2021Natur.593...33D">2021Natur.593...33D</a>. <a href="Doi_(identifier)" class="mw-redirect" title="Doi (identifier)">doi</a>:<a rel="nofollow" class="external text" href="https://doi.org/10.1038%2Fd41586-021-01170-0">10.1038/d41586-021-01170-0</a>. <a href="ISSN_(identifier)" class="mw-redirect" title="ISSN (identifier)">ISSN</a> <a rel="nofollow" class="external text" href="https://search.worldcat.org/issn/0028-0836">0028-0836</a>. <a href="PMID_(identifier)" class="mw-redirect" title="PMID (identifier)">PMID</a> <a rel="nofollow" class="external text" href="https://pubmed.ncbi.nlm.nih.gov/33947992">33947992</a>. <a href="S2CID_(identifier)" class="mw-redirect" title="S2CID (identifier)">S2CID</a> <a rel="nofollow" class="external text" href="https://api.semanticscholar.org/CorpusID:233740521">233740521</a>. <a rel="nofollow" class="external text" href="https://web.archive.org/web/20221218210857/https://www.nature.com/articles/d41586-021-01170-0">Archived</a> from the original on December 18, 2022. Retrieved September 12, 2022.</cite></li></ul>

</li>

<li id="cite_note-34"><a href="#cite_ref-34">^</a> <cite id="CITEREFPrunklWhittlestone2020" class="citation book cs1">Prunkl, Carina; Whittlestone, Jess (February 7, 2020). <a rel="nofollow" class="external text" href="https://dl.acm.org/doi/10.1145/3375627.3375803">"Beyond Near- and Long-Term"</a>. Proceedings of the AAAI/ACM Conference on AI, Ethics, and Society. New York NY USA: ACM. pp. 138–143. <a href="Doi_(identifier)" class="mw-redirect" title="Doi (identifier)">doi</a>:<a rel="nofollow" class="external text" href="https://doi.org/10.1145%2F3375627.3375803">10.1145/3375627.3375803</a>. <a href="ISBN_(identifier)" class="mw-redirect" title="ISBN (identifier)">ISBN</a> <bdi>978-1-4503-7110-0</bdi>. <a href="S2CID_(identifier)" class="mw-redirect" title="S2CID (identifier)">S2CID</a> <a rel="nofollow" class="external text" href="https://api.semanticscholar.org/CorpusID:210164673">210164673</a>. <a rel="nofollow" class="external text" href="https://web.archive.org/web/20221016123733/https://dl.acm.org/doi/10.1145/3375627.3375803">Archived</a> from the original on October 16, 2022. Retrieved September 12, 2022.</cite>

</li>

<li id="cite_note-:4-35">^ <a href="#cite_ref-:4_35-0">a</a> <a href="#cite_ref-:4_35-1">b</a> <a href="#cite_ref-:4_35-2">c</a> <a href="#cite_ref-:4_35-3">d</a> <cite id="CITEREFIrvingAskell2019" class="citation journal cs1">Irving, Geoffrey; Askell, Amanda (February 19, 2019). <a rel="nofollow" class="external text" href="https://distill.pub/2019/safety-needs-social-scientists">"AI Safety Needs Social Scientists"</a>. Distill. 4 (2): 10.23915/distill.00014. <a href="Doi_(identifier)" class="mw-redirect" title="Doi (identifier)">doi</a>:<a rel="nofollow" class="external text" href="https://doi.org/10.23915%2Fdistill.00014">10.23915/distill.00014</a>. <a href="ISSN_(identifier)" class="mw-redirect" title="ISSN (identifier)">ISSN</a> <a rel="nofollow" class="external text" href="https://search.worldcat.org/issn/2476-0757">2476-0757</a>. <a href="S2CID_(identifier)" class="mw-redirect" title="S2CID (identifier)">S2CID</a> <a rel="nofollow" class="external text" href="https://api.semanticscholar.org/CorpusID:159180422">159180422</a>. <a rel="nofollow" class="external text" href="https://web.archive.org/web/20230210114220/https://distill.pub/2019/safety-needs-social-scientists/">Archived</a> from the original on February 10, 2023. Retrieved September 12, 2022.</cite>

</li>

<li id="cite_note-36"><a href="#cite_ref-36">^</a> <cite id="CITEREFGazosKahnKuscheBüscher2025" class="citation journal cs1">Gazos, Alexandros; Kahn, James; Kusche, Isabel; Büscher, Christian; Götz, Markus (April 1, 2025). <a rel="nofollow" class="external text" href="https://doi.org/10.1016%2Fj.ssci.2024.106731">"Organising AI for safety: Identifying structural vulnerabilities to guide the design of AI-enhanced socio-technical systems"</a>. Safety Science. 184 106731. <a href="Doi_(identifier)" class="mw-redirect" title="Doi (identifier)">doi</a>:<a rel="nofollow" class="external text" href="https://doi.org/10.1016%2Fj.ssci.2024.106731">10.1016/j.ssci.2024.106731</a>. <a href="ISSN_(identifier)" class="mw-redirect" title="ISSN (identifier)">ISSN</a> <a rel="nofollow" class="external text" href="https://search.worldcat.org/issn/0925-7535">0925-7535</a>.</cite>

</li>

<li id="cite_note-40"><a href="#cite_ref-40">^</a> Bringsjord, Selmer and Govindarajulu, Naveen Sundar, <a rel="nofollow" class="external text" href="https://plato.stanford.edu/archives/sum2020/entries/artificial-intelligence/">"Artificial Intelligence"</a>, The Stanford Encyclopedia of Philosophy (Summer 2020 Edition), Edward N. Zalta (ed.)

</li>

<li id="cite_note-quanta_alphazero-41"><a href="#cite_ref-quanta_alphazero_41-0">^</a> <cite class="citation news cs1"><a rel="nofollow" class="external text" href="https://www.quantamagazine.org/why-alphazeros-artificial-intelligence-has-trouble-with-the-real-world-20180221/">"Why AlphaZero's Artificial Intelligence Has Trouble With the Real World"</a>. Quanta Magazine. 2018. Retrieved June 20, 2020.</cite>

</li>

<li id="cite_note-quanta_problem-42"><a href="#cite_ref-quanta_problem_42-0">^</a> <cite id="CITEREFWolchover2020" class="citation news cs1">Wolchover, Natalie (January 30, 2020). <a rel="nofollow" class="external text" href="https://www.quantamagazine.org/artificial-intelligence-will-do-what-we-ask-thats-a-problem-20200130/">"Artificial Intelligence Will Do What We Ask. That's a Problem"</a>. Quanta Magazine. Retrieved June 21, 2020.</cite>

</li>

<li id="cite_note-43"><a href="#cite_ref-43">^</a> Bull, Larry. "On model-based evolutionary computation." Soft Computing 3, no. 2 (1999): 76–82.

</li>

<li id="cite_note-Wiener1960-44">^ <a href="#cite_ref-Wiener1960_44-0">a</a> <a href="#cite_ref-Wiener1960_44-1">b</a> <cite id="CITEREFWiener1960" class="citation journal cs1">Wiener, Norbert (May 6, 1960). <a rel="nofollow" class="external text" href="https://www.science.org/doi/10.1126/science.131.3410.1355">"Some Moral and Technical Consequences of Automation: As machines learn they may develop unforeseen strategies at rates that baffle their programmers"</a>. Science. 131 (3410): 1355–1358. <a href="Doi_(identifier)" class="mw-redirect" title="Doi (identifier)">doi</a>:<a rel="nofollow" class="external text" href="https://doi.org/10.1126%2Fscience.131.3410.1355">10.1126/science.131.3410.1355</a>. <a href="ISSN_(identifier)" class="mw-redirect" title="ISSN (identifier)">ISSN</a> <a rel="nofollow" class="external text" href="https://search.worldcat.org/issn/0036-8075">0036-8075</a>. <a href="PMID_(identifier)" class="mw-redirect" title="PMID (identifier)">PMID</a> <a rel="nofollow" class="external text" href="https://pubmed.ncbi.nlm.nih.gov/17841602">17841602</a>. <a href="S2CID_(identifier)" class="mw-redirect" title="S2CID (identifier)">S2CID</a> <a rel="nofollow" class="external text" href="https://api.semanticscholar.org/CorpusID:30855376">30855376</a>. <a rel="nofollow" class="external text" href="https://web.archive.org/web/20221015105034/https://www.science.org/doi/10.1126/science.131.3410.1355">Archived</a> from the original on October 15, 2022. Retrieved September 12, 2022.</cite>

</li>

<li id="cite_note-Gabriel2020-45">^ <a href="#cite_ref-Gabriel2020_45-0">a</a> <a href="#cite_ref-Gabriel2020_45-1">b</a> <a href="#cite_ref-Gabriel2020_45-2">c</a> <a href="#cite_ref-Gabriel2020_45-3">d</a> <cite id="CITEREFGabriel2020" class="citation journal cs1">Gabriel, Iason (September 1, 2020). <a rel="nofollow" class="external text" href="https://doi.org/10.1007%2Fs11023-020-09539-2">"Artificial Intelligence, Values, and Alignment"</a>. Minds and Machines. 30 (3): 411–437. <a href="ArXiv_(identifier)" class="mw-redirect" title="ArXiv (identifier)">arXiv</a>:<a rel="nofollow" class="external text" href="https://arxiv.org/abs/2001.09768">2001.09768</a>. <a href="Doi_(identifier)" class="mw-redirect" title="Doi (identifier)">doi</a>:<a rel="nofollow" class="external text" href="https://doi.org/10.1007%2Fs11023-020-09539-2">10.1007/s11023-020-09539-2</a>. <a href="ISSN_(identifier)" class="mw-redirect" title="ISSN (identifier)">ISSN</a> <a rel="nofollow" class="external text" href="https://search.worldcat.org/issn/1572-8641">1572-8641</a>. <a href="S2CID_(identifier)" class="mw-redirect" title="S2CID (identifier)">S2CID</a> <a rel="nofollow" class="external text" href="https://api.semanticscholar.org/CorpusID:210920551">210920551</a>.</cite>

</li>

<li id="cite_note-46"><a href="#cite_ref-46">^</a> <cite id="CITEREFThe_Ezra_Klein_Show2021" class="citation news cs1">The Ezra Klein Show (June 4, 2021). <a rel="nofollow" class="external text" href="https://www.nytimes.com/2021/06/04/opinion/ezra-klein-podcast-brian-christian.html">"If 'All Models Are Wrong,' Why Do We Give Them So Much Power?"</a>. The New York Times. <a href="ISSN_(identifier)" class="mw-redirect" title="ISSN (identifier)">ISSN</a> <a rel="nofollow" class="external text" href="https://search.worldcat.org/issn/0362-4331">0362-4331</a>. <a rel="nofollow" class="external text" href="https://web.archive.org/web/20230215224050/https://www.nytimes.com/2021/06/04/opinion/ezra-klein-podcast-brian-christian.html">Archived</a> from the original on February 15, 2023. Retrieved March 13, 2023.</cite>

<ul><li><cite id="CITEREFWolchover2015" class="citation web cs1">Wolchover, Natalie (April 21, 2015). <a rel="nofollow" class="external text" href="https://www.quantamagazine.org/artificial-intelligence-aligned-with-human-values-qa-with-stuart-russell-20150421/">"Concerns of an Artificial Intelligence Pioneer"</a>. Quanta Magazine. <a rel="nofollow" class="external text" href="https://web.archive.org/web/20230210114137/https://www.quantamagazine.org/artificial-intelligence-aligned-with-human-values-qa-with-stuart-russell-20150421/">Archived</a> from the original on February 10, 2023. Retrieved March 13, 2023.</cite></li>

<li><cite id="CITEREFCalifornia_Assembly" class="citation web cs1">California Assembly. <a rel="nofollow" class="external text" href="https://leginfo.legislature.ca.gov/faces/billTextClient.xhtml?bill_id=201720180ACR215">"Bill Text – ACR-215 23 Asilomar AI Principles"</a>. <a rel="nofollow" class="external text" href="https://web.archive.org/web/20230210114137/https://leginfo.legislature.ca.gov/faces/billTextClient.xhtml?bill_id=201720180ACR215">Archived</a> from the original on February 10, 2023. Retrieved July 18, 2022.</cite></li></ul>

</li>

<li id="cite_note-MasteringLanguage-47">^ <a href="#cite_ref-MasteringLanguage_47-0">a</a> <a href="#cite_ref-MasteringLanguage_47-1">b</a> <cite id="CITEREFJohnsonIziev2022" class="citation news cs1">Johnson, Steven; Iziev, Nikita (April 15, 2022). <a rel="nofollow" class="external text" href="https://www.nytimes.com/2022/04/15/magazine/ai-language.html">"A.I. Is Mastering Language. Should We Trust What It Says?"</a>. The New York Times. <a href="ISSN_(identifier)" class="mw-redirect" title="ISSN (identifier)">ISSN</a> <a rel="nofollow" class="external text" href="https://search.worldcat.org/issn/0362-4331">0362-4331</a>. <a rel="nofollow" class="external text" href="https://web.archive.org/web/20221124151408/https://www.nytimes.com/2022/04/15/magazine/ai-language.html">Archived</a> from the original on November 24, 2022. Retrieved July 18, 2022.</cite>

</li>

<li id="cite_note-48"><a href="#cite_ref-48">^</a> <cite id="CITEREFOpenAI" class="citation web cs1">OpenAI. <a rel="nofollow" class="external text" href="https://openai.com/blog/our-approach-to-alignment-research">"Developing safe & responsible AI"</a>. Retrieved March 13, 2023.</cite>

<ul><li><cite class="citation web cs1"><a rel="nofollow" class="external text" href="https://deepmindsafetyresearch.medium.com">"DeepMind Safety Research"</a>. Medium. <a rel="nofollow" class="external text" href="https://web.archive.org/web/20230210114142/https://deepmindsafetyresearch.medium.com/">Archived</a> from the original on February 10, 2023. Retrieved March 13, 2023.</cite></li></ul>

</li>

<li id="cite_note-Unsolved2022-49">^ <a href="#cite_ref-Unsolved2022_49-0">a</a> <a href="#cite_ref-Unsolved2022_49-1">b</a> <a href="#cite_ref-Unsolved2022_49-2">c</a> <a href="#cite_ref-Unsolved2022_49-3">d</a> <a href="#cite_ref-Unsolved2022_49-4">e</a> <a href="#cite_ref-Unsolved2022_49-5">f</a> <cite id="CITEREFHendrycksCarliniSchulmanSteinhardt2022" class="citation arxiv cs1">Hendrycks, Dan; <a href="Nicholas_Carlini" title="Nicholas Carlini">Carlini, Nicholas</a>; Schulman, John; Steinhardt, Jacob (June 16, 2022). "Unsolved Problems in ML Safety". <a href="ArXiv_(identifier)" class="mw-redirect" title="ArXiv (identifier)">arXiv</a>:<a rel="nofollow" class="external text" href="https://arxiv.org/abs/2109.13916">2109.13916</a> [<a rel="nofollow" class="external text" href="https://arxiv.org/archive/cs.LG">cs.LG</a>].</cite>

</li>

<li id="cite_note-50"><a href="#cite_ref-50">^</a> <cite id="CITEREFRussellNorvig2022" class="citation book cs1">Russell, Stuart J.; Norvig, Peter (2022). <a rel="nofollow" class="external text" href="https://www.pearson.com/us/higher-education/program/Russell-Artificial-Intelligence-A-Modern-Approach-4th-Edition/PGM1263338.html">Artificial intelligence: a modern approach</a> (4th ed.). Pearson. pp. 4–5. <a href="ISBN_(identifier)" class="mw-redirect" title="ISBN (identifier)">ISBN</a> <bdi>978-1-292-40113-3</bdi>. <a href="OCLC_(identifier)" class="mw-redirect" title="OCLC (identifier)">OCLC</a> <a rel="nofollow" class="external text" href="https://search.worldcat.org/oclc/1303900751">1303900751</a>.</cite>

</li>

<li id="cite_note-SpecGaming2020-51">^ <a href="#cite_ref-SpecGaming2020_51-0">a</a> <a href="#cite_ref-SpecGaming2020_51-1">b</a> <a href="#cite_ref-SpecGaming2020_51-2">c</a> <a href="#cite_ref-SpecGaming2020_51-3">d</a> <a href="#cite_ref-SpecGaming2020_51-4">e</a> <cite id="CITEREFKrakovnaUesatoMikulikRahtz2020" class="citation web cs1">Krakovna, Victoria; Uesato, Jonathan; Mikulik, Vladimir; Rahtz, Matthew; Everitt, Tom; Kumar, Ramana; Kenton, Zac; Leike, Jan; Legg, Shane (April 21, 2020). <a rel="nofollow" class="external text" href="https://www.deepmind.com/blog/specification-gaming-the-flip-side-of-ai-ingenuity">"Specification gaming: the flip side of AI ingenuity"</a>. Deepmind. <a rel="nofollow" class="external text" href="https://web.archive.org/web/20230210114143/https://www.deepmind.com/blog/specification-gaming-the-flip-side-of-ai-ingenuity">Archived</a> from the original on February 10, 2023. Retrieved August 26, 2022.</cite>

</li>

<li id="cite_note-:111-52"><a href="#cite_ref-:111_52-0">^</a> <cite id="CITEREFManheimGarrabrant2018" class="citation arxiv cs1">Manheim, David; Garrabrant, Scott (2018). "Categorizing Variants of Goodhart's Law". <a href="ArXiv_(identifier)" class="mw-redirect" title="ArXiv (identifier)">arXiv</a>:<a rel="nofollow" class="external text" href="https://arxiv.org/abs/1803.04585">1803.04585</a> [<a rel="nofollow" class="external text" href="https://arxiv.org/archive/cs.AI">cs.AI</a>].</cite>

</li>

<li id="cite_note-lfhp2017-53">^ <a href="#cite_ref-lfhp2017_53-0">a</a> <a href="#cite_ref-lfhp2017_53-1">b</a> <a href="#cite_ref-lfhp2017_53-2">c</a> <cite id="CITEREFAmodeiChristianoRay2017" class="citation web cs1">Amodei, Dario; Christiano, Paul; Ray, Alex (June 13, 2017). <a rel="nofollow" class="external text" href="https://openai.com/blog/deep-reinforcement-learning-from-human-preferences/">"Learning from Human Preferences"</a>. OpenAI. <a rel="nofollow" class="external text" href="https://web.archive.org/web/20210103215933/https://openai.com/blog/deep-reinforcement-learning-from-human-preferences/">Archived</a> from the original on January 3, 2021. Retrieved July 21, 2022.</cite>

</li>

<li id="cite_note-54"><a href="#cite_ref-54">^</a> <cite class="citation web cs1"><a rel="nofollow" class="external text" href="https://docs.google.com/spreadsheets/d/e/2PACX-1vRPiprOaC3HsCf5Tuum8bRfzYUiKLRqJmbOoC-32JorNdfyTiRRsR7Ea5eWtvsWzuxo8bjOxCG84dAg/pubhtml">"Specification gaming examples in AI - master list - Google Drive"</a>. docs.google.com.</cite>

</li>

<li id="cite_note-55"><a href="#cite_ref-55">^</a> <cite id="CITEREFClarkAmodei2016" class="citation web cs1">Clark, Jack; Amodei, Dario (December 21, 2016). <a rel="nofollow" class="external text" href="https://openai.com/research/faulty-reward-functions">"Faulty reward functions in the wild"</a>. openai.com. Retrieved December 30, 2023.</cite>

</li>

<li id="cite_note-TruthfulQA-56">^ <a href="#cite_ref-TruthfulQA_56-0">a</a> <a href="#cite_ref-TruthfulQA_56-1">b</a> <a href="#cite_ref-TruthfulQA_56-2">c</a> <cite id="CITEREFLinHiltonEvans2022" class="citation journal cs1">Lin, Stephanie; Hilton, Jacob; Evans, Owain (2022). <a rel="nofollow" class="external text" href="https://aclanthology.org/2022.acl-long.229">"TruthfulQA: Measuring How Models Mimic Human Falsehoods"</a>. Proceedings of the 60th Annual Meeting of the Association for Computational Linguistics (Volume 1: Long Papers). Dublin, Ireland: Association for Computational Linguistics: 3214–3252. <a href="ArXiv_(identifier)" class="mw-redirect" title="ArXiv (identifier)">arXiv</a>:<a rel="nofollow" class="external text" href="https://arxiv.org/abs/2109.07958">2109.07958</a>. <a href="Doi_(identifier)" class="mw-redirect" title="Doi (identifier)">doi</a>:<a rel="nofollow" class="external text" href="https://doi.org/10.18653%2Fv1%2F2022.acl-long.229">10.18653/v1/2022.acl-long.229</a>. <a href="S2CID_(identifier)" class="mw-redirect" title="S2CID (identifier)">S2CID</a> <a rel="nofollow" class="external text" href="https://api.semanticscholar.org/CorpusID:237532606">237532606</a>. <a rel="nofollow" class="external text" href="https://web.archive.org/web/20230210114231/https://aclanthology.org/2022.acl-long.229/">Archived</a> from the original on February 10, 2023. Retrieved September 12, 2022.</cite>

</li>

<li id="cite_note-Naughton2021-57">^ <a href="#cite_ref-Naughton2021_57-0">a</a> <a href="#cite_ref-Naughton2021_57-1">b</a> <a href="#cite_ref-Naughton2021_57-2">c</a> <cite id="CITEREFNaughton2021" class="citation news cs1">Naughton, John (October 2, 2021). <a rel="nofollow" class="external text" href="https://www.theguardian.com/commentisfree/2021/oct/02/the-truth-about-artificial-intelligence-it-isnt-that-honest">"The truth about artificial intelligence? It isn't that honest"</a>. The Observer. <a href="ISSN_(identifier)" class="mw-redirect" title="ISSN (identifier)">ISSN</a> <a rel="nofollow" class="external text" href="https://search.worldcat.org/issn/0029-7712">0029-7712</a>. <a rel="nofollow" class="external text" href="https://web.archive.org/web/20230213231317/https://www.theguardian.com/commentisfree/2021/oct/02/the-truth-about-artificial-intelligence-it-isnt-that-honest">Archived</a> from the original on February 13, 2023. Retrieved July 23, 2022.</cite>

</li>

<li id="cite_note-58"><a href="#cite_ref-58">^</a> <cite id="CITEREFJiLeeFrieskeYu2022" class="citation journal cs1">Ji, Ziwei; Lee, Nayeon; Frieske, Rita; Yu, Tiezheng; Su, Dan; Xu, Yan; Ishii, Etsuko; Bang, Yejin; Madotto, Andrea; Fung, Pascale (February 1, 2022). "Survey of Hallucination in Natural Language Generation". ACM Computing Surveys. 55 (12): 1–38. <a href="ArXiv_(identifier)" class="mw-redirect" title="ArXiv (identifier)">arXiv</a>:<a rel="nofollow" class="external text" href="https://arxiv.org/abs/2202.03629">2202.03629</a>. <a href="Doi_(identifier)" class="mw-redirect" title="Doi (identifier)">doi</a>:<a rel="nofollow" class="external text" href="https://doi.org/10.1145%2F3571730">10.1145/3571730</a>. <a href="S2CID_(identifier)" class="mw-redirect" title="S2CID (identifier)">S2CID</a> <a rel="nofollow" class="external text" href="https://api.semanticscholar.org/CorpusID:246652372">246652372</a>.</cite>

<ul><li><cite id="CITEREFElse2023" class="citation journal cs1">Else, Holly (January 12, 2023). <a rel="nofollow" class="external text" href="https://www.nature.com/articles/d41586-023-00056-7">"Abstracts written by ChatGPT fool scientists"</a>. Nature. 613 (7944): 423. <a href="Bibcode_(identifier)" class="mw-redirect" title="Bibcode (identifier)">Bibcode</a>:<a rel="nofollow" class="external text" href="https://ui.adsabs.harvard.edu/abs/2023Natur.613..423E">2023Natur.613..423E</a>. <a href="Doi_(identifier)" class="mw-redirect" title="Doi (identifier)">doi</a>:<a rel="nofollow" class="external text" href="https://doi.org/10.1038%2Fd41586-023-00056-7">10.1038/d41586-023-00056-7</a>. <a href="PMID_(identifier)" class="mw-redirect" title="PMID (identifier)">PMID</a> <a rel="nofollow" class="external text" href="https://pubmed.ncbi.nlm.nih.gov/36635510">36635510</a>. <a href="S2CID_(identifier)" class="mw-redirect" title="S2CID (identifier)">S2CID</a> <a rel="nofollow" class="external text" href="https://api.semanticscholar.org/CorpusID:255773668">255773668</a>.</cite></li></ul>

</li>

<li id="cite_note-:5-59"><a href="#cite_ref-:5_59-0">^</a> <cite id="CITEREFRussell" class="citation web cs1">Russell, Stuart. <a rel="nofollow" class="external text" href="https://www.edge.org/conversation/the-myth-of-ai">"Of Myths and Moonshine"</a>. Edge.org. <a rel="nofollow" class="external text" href="https://web.archive.org/web/20230210114137/https://www.edge.org/conversation/the-myth-of-ai">Archived</a> from the original on February 10, 2023. Retrieved July 19, 2022.</cite>

</li>

<li id="cite_note-60"><a href="#cite_ref-60">^</a> <cite id="CITEREFTasioulas2019" class="citation journal cs1">Tasioulas, John (2019). "First Steps Towards an Ethics of Robots and Artificial Intelligence". Journal of Practical Ethics. 7 (1): 61–95.</cite>

</li>

<li id="cite_note-61"><a href="#cite_ref-61">^</a> <cite id="CITEREFBooth2025" class="citation magazine cs1">Booth, Harry (February 19, 2025). <a rel="nofollow" class="external text" href="https://time.com/7259395/ai-chess-cheating-palisade-research/">"When AI Thinks It Will Lose, It Sometimes Cheats"</a>. TIME. Retrieved February 23, 2025.</cite>

</li>

<li id="cite_note-:722-62"><a href="#cite_ref-:722_62-0">^</a> <cite id="CITEREFWellsDeepa_SeetharamanHorwitz2021" class="citation news cs1">Wells, Georgia; Deepa Seetharaman; Horwitz, Jeff (November 5, 2021). <a rel="nofollow" class="external text" href="https://www.wsj.com/articles/facebook-bad-for-you-360-million-users-say-yes-company-documents-facebook-files-11636124681">"Is Facebook Bad for You? It Is for About 360 Million Users, Company Surveys Suggest"</a>. The Wall Street Journal. <a href="ISSN_(identifier)" class="mw-redirect" title="ISSN (identifier)">ISSN</a> <a rel="nofollow" class="external text" href="https://search.worldcat.org/issn/0099-9660">0099-9660</a>. <a rel="nofollow" class="external text" href="https://web.archive.org/web/20230210114137/https://www.wsj.com/articles/facebook-bad-for-you-360-million-users-say-yes-company-documents-facebook-files-11636124681">Archived</a> from the original on February 10, 2023. Retrieved July 19, 2022.</cite>

</li>

<li id="cite_note-:822-63"><a href="#cite_ref-:822_63-0">^</a> <cite id="CITEREFBarrettHendrixSims2021" class="citation report cs1">Barrett, Paul M.; Hendrix, Justin; Sims, J. Grant (September 2021). <a rel="nofollow" class="external text" href="https://bhr.stern.nyu.edu/polarization-report-page">How Social Media Intensifies U.S. Political Polarization-And What Can Be Done About It</a> (Report). Center for Business and Human Rights, NYU. <a rel="nofollow" class="external text" href="https://web.archive.org/web/20230201180005/https://bhr.stern.nyu.edu/polarization-report-page">Archived</a> from the original on February 1, 2023. Retrieved September 12, 2022.</cite>

</li>

<li id="cite_note-64"><a href="#cite_ref-64">^</a> <cite id="CITEREFShepardson2018" class="citation news cs1">Shepardson, David (May 24, 2018). <a rel="nofollow" class="external text" href="https://www.reuters.com/article/us-uber-crash-idUSKCN1IP26K">"Uber disabled emergency braking in self-driving car: U.S. agency"</a>. Reuters. <a rel="nofollow" class="external text" href="https://web.archive.org/web/20230210114137/https://www.reuters.com/article/us-uber-crash-idUSKCN1IP26K">Archived</a> from the original on February 10, 2023. Retrieved July 20, 2022.</cite>

</li>

<li id="cite_note-65"><a href="#cite_ref-65">^</a> <cite class="citation web cs1"><a rel="nofollow" class="external text" href="https://www.technologyreview.com/2020/02/17/844721/ai-openai-moonshot-elon-musk-sam-altman-greg-brockman-messy-secretive-reality/">"The messy, secretive reality behind OpenAI's bid to save the world"</a>. MIT Technology Review. Retrieved August 25, 2024.</cite>

</li>

<li id="cite_note-66"><a href="#cite_ref-66">^</a> <cite id="CITEREFHeath2024" class="citation web cs1">Heath, Alex (January 18, 2024). <a rel="nofollow" class="external text" href="https://www.theverge.com/2024/1/18/24042354/mark-zuckerberg-meta-agi-reorg-interview">"Mark Zuckerberg's new goal is creating artificial general intelligence"</a>. The Verge. Retrieved November 5, 2024.</cite>

</li>

<li id="cite_note-67"><a href="#cite_ref-67">^</a> <cite id="CITEREFJohnson" class="citation web cs1">Johnson, Dave. <a rel="nofollow" class="external text" href="https://www.businessinsider.com/google-deepmind">"DeepMind is Google's AI research hub. Here's what it does, where it's located, and how it differs from OpenAI"</a>. Business Insider. Retrieved August 25, 2024.</cite>

</li>

<li id="cite_note-eallm2022-68">^ <a href="#cite_ref-eallm2022_68-0">a</a> <a href="#cite_ref-eallm2022_68-1">b</a> <cite id="CITEREFWeiTayBommasaniRaffel2022" class="citation journal cs1">Wei, Jason; Tay, Yi; Bommasani, Rishi; Raffel, Colin; Zoph, Barret; Borgeaud, Sebastian; Yogatama, Dani; Bosma, Maarten; Zhou, Denny; Metzler, Donald; Chi, Ed H.; Hashimoto, Tatsunori; Vinyals, Oriol; Liang, Percy; Dean, Jeff; Fedus, William (October 26, 2022). "Emergent Abilities of Large Language Models". Transactions on Machine Learning Research. <a href="ArXiv_(identifier)" class="mw-redirect" title="ArXiv (identifier)">arXiv</a>:<a rel="nofollow" class="external text" href="https://arxiv.org/abs/2206.07682">2206.07682</a>. <a href="ISSN_(identifier)" class="mw-redirect" title="ISSN (identifier)">ISSN</a> <a rel="nofollow" class="external text" href="https://search.worldcat.org/issn/2835-8856">2835-8856</a>.</cite>

</li>

<li id="cite_note-:0-69">^ <a href="#cite_ref-:0_69-0">a</a> <a href="#cite_ref-:0_69-1">b</a> <cite id="CITEREFCaballeroGuptaRishKrueger2022" class="citation arxiv cs1">Caballero, Ethan; Gupta, Kshitij; Rish, Irina; Krueger, David (2022). "Broken Neural Scaling Laws". <a href="ArXiv_(identifier)" class="mw-redirect" title="ArXiv (identifier)">arXiv</a>:<a rel="nofollow" class="external text" href="https://arxiv.org/abs/2210.14891">2210.14891</a> [<a rel="nofollow" class="external text" href="https://arxiv.org/archive/cs.LG">cs.LG</a>].</cite>

</li>

<li id="cite_note-70"><a href="#cite_ref-70">^</a> <cite id="CITEREFDominguez2022" class="citation web cs1">Dominguez, Daniel (May 19, 2022). <a rel="nofollow" class="external text" href="https://www.infoq.com/news/2022/05/deepmind-gato-ai-agent/">"DeepMind Introduces Gato, a New Generalist AI Agent"</a>. InfoQ. <a rel="nofollow" class="external text" href="https://web.archive.org/web/20230210114137/https://www.infoq.com/news/2022/05/deepmind-gato-ai-agent/">Archived</a> from the original on February 10, 2023. Retrieved September 9, 2022.</cite>

<ul><li><cite id="CITEREFEdwards2022" class="citation web cs1">Edwards, Ben (April 26, 2022). <a rel="nofollow" class="external text" href="https://arstechnica.com/information-technology/2022/09/new-ai-assistant-can-browse-search-and-use-web-apps-like-a-human/">"Adept's AI assistant can browse, search, and use web apps like a human"</a>. Ars Technica. <a rel="nofollow" class="external text" href="https://web.archive.org/web/20230117194921/https://arstechnica.com/information-technology/2022/09/new-ai-assistant-can-browse-search-and-use-web-apps-like-a-human/">Archived</a> from the original on January 17, 2023. Retrieved September 9, 2022.</cite></li></ul>

</li>

<li id="cite_note-71"><a href="#cite_ref-71">^</a> <cite id="CITEREFGraceStewartSandkühlerThomas2024" class="citation arxiv cs1">Grace, Katja; Stewart, Harlan; Sandkühler, Julia Fabienne; Thomas, Stephen; Weinstein-Raun, Ben; Brauner, Jan (January 5, 2024). "Thousands of AI Authors on the Future of AI". <a href="ArXiv_(identifier)" class="mw-redirect" title="ArXiv (identifier)">arXiv</a>:<a rel="nofollow" class="external text" href="https://arxiv.org/abs/2401.02843">2401.02843</a> [<a rel="nofollow" class="external text" href="https://arxiv.org/archive/cs.CY">cs.CY</a>].</cite>

</li>

<li id="cite_note-:2822-72"><a href="#cite_ref-:2822_72-0">^</a> <cite id="CITEREFGraceSalvatierDafoeZhang2018" class="citation journal cs1">Grace, Katja; Salvatier, John; Dafoe, Allan; Zhang, Baobao; Evans, Owain (July 31, 2018). <a rel="nofollow" class="external text" href="http://jair.org/index.php/jair/article/view/11222">"Viewpoint: When Will AI Exceed Human Performance? Evidence from AI Experts"</a>. Journal of Artificial Intelligence Research. 62: 729–754. <a href="Doi_(identifier)" class="mw-redirect" title="Doi (identifier)">doi</a>:<a rel="nofollow" class="external text" href="https://doi.org/10.1613%2Fjair.1.11222">10.1613/jair.1.11222</a>. <a href="ISSN_(identifier)" class="mw-redirect" title="ISSN (identifier)">ISSN</a> <a rel="nofollow" class="external text" href="https://search.worldcat.org/issn/1076-9757">1076-9757</a>. <a href="S2CID_(identifier)" class="mw-redirect" title="S2CID (identifier)">S2CID</a> <a rel="nofollow" class="external text" href="https://api.semanticscholar.org/CorpusID:8746462">8746462</a>. <a rel="nofollow" class="external text" href="https://web.archive.org/web/20230210114220/https://jair.org/index.php/jair/article/view/11222">Archived</a> from the original on February 10, 2023. Retrieved September 12, 2022.</cite>

</li>

<li id="cite_note-:2922-73"><a href="#cite_ref-:2922_73-0">^</a> <cite id="CITEREFZhangAnderljungKahnDreksler2021" class="citation journal cs1">Zhang, Baobao; Anderljung, Markus; Kahn, Lauren; Dreksler, Noemi; Horowitz, Michael C.; Dafoe, Allan (August 2, 2021). <a rel="nofollow" class="external text" href="https://jair.org/index.php/jair/article/view/12895">"Ethics and Governance of Artificial Intelligence: Evidence from a Survey of Machine Learning Researchers"</a>. Journal of Artificial Intelligence Research. 71. <a href="ArXiv_(identifier)" class="mw-redirect" title="ArXiv (identifier)">arXiv</a>:<a rel="nofollow" class="external text" href="https://arxiv.org/abs/2105.02117">2105.02117</a>. <a href="Doi_(identifier)" class="mw-redirect" title="Doi (identifier)">doi</a>:<a rel="nofollow" class="external text" href="https://doi.org/10.1613%2Fjair.1.12895">10.1613/jair.1.12895</a>. <a href="ISSN_(identifier)" class="mw-redirect" title="ISSN (identifier)">ISSN</a> <a rel="nofollow" class="external text" href="https://search.worldcat.org/issn/1076-9757">1076-9757</a>. <a href="S2CID_(identifier)" class="mw-redirect" title="S2CID (identifier)">S2CID</a> <a rel="nofollow" class="external text" href="https://api.semanticscholar.org/CorpusID:233740003">233740003</a>. <a rel="nofollow" class="external text" href="https://web.archive.org/web/20230210114143/https://jair.org/index.php/jair/article/view/12895">Archived</a> from the original on February 10, 2023. Retrieved September 12, 2022.</cite>

</li>

<li id="cite_note-:1701-74"><a href="#cite_ref-:1701_74-0">^</a> <cite id="CITEREFFuture_of_Life_Institute2023" class="citation web cs1">Future of Life Institute (March 22, 2023). <a rel="nofollow" class="external text" href="https://futureoflife.org/open-letter/pause-giant-ai-experiments/">"Pause Giant AI Experiments: An Open Letter"</a>. Retrieved April 20, 2023.</cite>

</li>

<li id="cite_note-75"><a href="#cite_ref-75">^</a> <cite id="CITEREFWangMaFengZhang2024" class="citation journal cs1">Wang, Lei; Ma, Chen; Feng, Xueyang; Zhang, Zeyu; Yang, Hao; Zhang, Jingsen; Chen, Zhiyuan; Tang, Jiakai; Chen, Xu (2024). <a rel="nofollow" class="external text" href="https://ui.adsabs.harvard.edu/abs/2023arXiv230811432W">"A survey on large language model based autonomous agents"</a>. Frontiers of Computer Science. 18 (6) 186345. <a href="ArXiv_(identifier)" class="mw-redirect" title="ArXiv (identifier)">arXiv</a>:<a rel="nofollow" class="external text" href="https://arxiv.org/abs/2308.11432">2308.11432</a>. <a href="Doi_(identifier)" class="mw-redirect" title="Doi (identifier)">doi</a>:<a rel="nofollow" class="external text" href="https://doi.org/10.1007%2Fs11704-024-40231-1">10.1007/s11704-024-40231-1</a>. Retrieved February 11, 2024.</cite>

</li>

<li id="cite_note-76"><a href="#cite_ref-76">^</a> <cite id="CITEREFBerglundSticklandBalesniKaufmann2023" class="citation arxiv cs1">Berglund, Lukas; Stickland, Asa Cooper; Balesni, Mikita; Kaufmann, Max; Tong, Meg; Korbak, Tomasz; Kokotajlo, Daniel; Evans, Owain (September 1, 2023). "Taken out of context: On measuring situational awareness in LLMs". <a href="ArXiv_(identifier)" class="mw-redirect" title="ArXiv (identifier)">arXiv</a>:<a rel="nofollow" class="external text" href="https://arxiv.org/abs/2309.00667">2309.00667</a> [<a rel="nofollow" class="external text" href="https://arxiv.org/archive/cs.CL">cs.CL</a>].</cite>

</li>

<li id="cite_note-77"><a href="#cite_ref-77">^</a> <cite id="CITEREFLaineMeinkeEvans2023" class="citation journal cs1">Laine, Rudolf; Meinke, Alexander; Evans, Owain (November 28, 2023). <a rel="nofollow" class="external text" href="https://openreview.net/forum?id=DRk4bWKr41&referrer=%5Bthe+profile+of+Rudolf+Laine%5D(/profile?id=~Rudolf_Laine1)">"Towards a Situational Awareness Benchmark for LLMs"</a>. NeurIPS 2023 SoLaR Workshop.</cite>

</li>

<li id="cite_note-:3-78">^ <a href="#cite_ref-:3_78-0">a</a> <a href="#cite_ref-:3_78-1">b</a> <cite id="CITEREFPanShernZouLi2023" class="citation journal cs1">Pan, Alexander; Shern, Chan Jun; Zou, Andy; Li, Nathaniel; Basart, Steven; Woodside, Thomas; Ng, Jonathan; Zhang, Emmons; Scott, Dan; Hendrycks (April 3, 2023). "Do the Rewards Justify the Means? Measuring Trade-Offs Between Rewards and Ethical Behavior in the MACHIAVELLI Benchmark". Proceedings of the 40th International Conference on Machine Learning. PMLR. <a href="ArXiv_(identifier)" class="mw-redirect" title="ArXiv (identifier)">arXiv</a>:<a rel="nofollow" class="external text" href="https://arxiv.org/abs/2304.03279">2304.03279</a>.</cite>

</li>

<li id="cite_note-dllmmwe2022-79">^ <a href="#cite_ref-dllmmwe2022_79-0">a</a> <a href="#cite_ref-dllmmwe2022_79-1">b</a> <a href="#cite_ref-dllmmwe2022_79-2">c</a> <a href="#cite_ref-dllmmwe2022_79-3">d</a> <cite id="CITEREFPerezRingerLukošiūtėNguyen2022" class="citation arxiv cs1">Perez, Ethan; Ringer, Sam; Lukošiūtė, Kamilė; Nguyen, Karina; Chen, Edwin; Heiner, Scott; Pettit, Craig; Olsson, Catherine; Kundu, Sandipan; Kadavath, Saurav; Jones, Andy; Chen, Anna; Mann, Ben; Israel, Brian; Seethor, Bryan (December 19, 2022). "Discovering Language Model Behaviors with Model-Written Evaluations". <a href="ArXiv_(identifier)" class="mw-redirect" title="ArXiv (identifier)">arXiv</a>:<a rel="nofollow" class="external text" href="https://arxiv.org/abs/2212.09251">2212.09251</a> [<a rel="nofollow" class="external text" href="https://arxiv.org/archive/cs.CL">cs.CL</a>].</cite>

</li>

<li id="cite_note-80"><a href="#cite_ref-80">^</a> <cite id="CITEREFOrseauArmstrong2016" class="citation journal cs1">Orseau, Laurent; Armstrong, Stuart (June 25, 2016). <a rel="nofollow" class="external text" href="https://dl.acm.org/doi/abs/10.5555/3020948.3021006">"Safely interruptible agents"</a>. Proceedings of the Thirty-Second Conference on Uncertainty in Artificial Intelligence. UAI'16. Arlington, Virginia, USA: AUAI Press: 557–566. <a href="ISBN_(identifier)" class="mw-redirect" title="ISBN (identifier)">ISBN</a> <bdi>978-0-9966431-1-5</bdi>.</cite>

</li>

<li id="cite_note-Gridworlds-81">^ <a href="#cite_ref-Gridworlds_81-0">a</a> <a href="#cite_ref-Gridworlds_81-1">b</a> <cite id="CITEREFLeikeMarticKrakovnaOrtega2017" class="citation arxiv cs1">Leike, Jan; Martic, Miljan; Krakovna, Victoria; Ortega, Pedro A.; Everitt, Tom; Lefrancq, Andrew; Orseau, Laurent; Legg, Shane (November 28, 2017). "AI Safety Gridworlds". <a href="ArXiv_(identifier)" class="mw-redirect" title="ArXiv (identifier)">arXiv</a>:<a rel="nofollow" class="external text" href="https://arxiv.org/abs/1711.09883">1711.09883</a> [<a rel="nofollow" class="external text" href="https://arxiv.org/archive/cs.LG">cs.LG</a>].</cite>

</li>

<li id="cite_note-OffSwitch-82">^ <a href="#cite_ref-OffSwitch_82-0">a</a> <a href="#cite_ref-OffSwitch_82-1">b</a> <a href="#cite_ref-OffSwitch_82-2">c</a> <a href="#cite_ref-OffSwitch_82-3">d</a> <cite id="CITEREFHadfield-MenellDraganAbbeelRussell2017" class="citation journal cs1">Hadfield-Menell, Dylan; Dragan, Anca; Abbeel, Pieter; Russell, Stuart (August 19, 2017). <a rel="nofollow" class="external text" href="https://dl.acm.org/doi/10.5555/3171642.3171675">"The off-switch game"</a>. Proceedings of the 26th International Joint Conference on Artificial Intelligence. IJCAI'17. Melbourne, Australia: AAAI Press: 220–227. <a href="ISBN_(identifier)" class="mw-redirect" title="ISBN (identifier)">ISBN</a> <bdi>978-0-9992411-0-3</bdi>.</cite>

</li>

<li id="cite_note-optsp-83">^ <a href="#cite_ref-optsp_83-0">a</a> <a href="#cite_ref-optsp_83-1">b</a> <a href="#cite_ref-optsp_83-2">c</a> <a href="#cite_ref-optsp_83-3">d</a> <cite id="CITEREFTurnerSmithShahCritch2021" class="citation conference cs1">Turner, Alexander Matt; Smith, Logan Riggs; Shah, Rohin; Critch, Andrew; Tadepalli, Prasad (2021). <a rel="nofollow" class="external text" href="https://openreview.net/forum?id=l7-DBWawSZH">"Optimal policies tend to seek power"</a>. Advances in neural information processing systems.</cite>

</li>

<li id="cite_note-84"><a href="#cite_ref-84">^</a> <cite id="CITEREFTurnerTadepalli2022" class="citation conference cs1">Turner, Alexander Matt; Tadepalli, Prasad (2022). <a rel="nofollow" class="external text" href="https://openreview.net/forum?id=GFgjnk2Q-ju">"Parametrically retargetable decision-makers tend to seek power"</a>. Advances in neural information processing systems.</cite>

</li>

<li id="cite_note-Superintelligence-85">^ <a href="#cite_ref-Superintelligence_85-0">a</a> <a href="#cite_ref-Superintelligence_85-1">b</a> <a href="#cite_ref-Superintelligence_85-2">c</a> <a href="#cite_ref-Superintelligence_85-3">d</a> <a href="#cite_ref-Superintelligence_85-4">e</a> <cite id="CITEREFBostrom2014" class="citation book cs1">Bostrom, Nick (2014). Superintelligence: Paths, Dangers, Strategies (1st ed.). USA: Oxford University Press, Inc. <a href="ISBN_(identifier)" class="mw-redirect" title="ISBN (identifier)">ISBN</a> <bdi>978-0-19-967811-2</bdi>.</cite>

</li>

<li id="cite_note-:1-86">^ <a href="#cite_ref-:1_86-0">a</a> <a href="#cite_ref-:1_86-1">b</a> <cite class="citation web cs1"><a rel="nofollow" class="external text" href="https://www.safe.ai/statement-on-ai-risk">"Statement on AI Risk | CAIS"</a>. www.safe.ai. Retrieved July 17, 2023.</cite>

</li>

<li id="cite_note-87"><a href="#cite_ref-87">^</a> <cite id="CITEREFRoose2023" class="citation news cs1">Roose, Kevin (May 30, 2023). <a rel="nofollow" class="external text" href="https://www.nytimes.com/2023/05/30/technology/ai-threat-warning.html">"A.I. Poses 'Risk of Extinction,' Industry Leaders Warn"</a>. The New York Times. <a href="ISSN_(identifier)" class="mw-redirect" title="ISSN (identifier)">ISSN</a> <a rel="nofollow" class="external text" href="https://search.worldcat.org/issn/0362-4331">0362-4331</a>. Retrieved July 17, 2023.</cite>

</li>

<li id="cite_note-88"><a href="#cite_ref-88">^</a> <cite id="CITEREFTuring1951" class="citation speech cs1">Turing, Alan (1951). <a rel="nofollow" class="external text" href="https://turingarchive.kings.cam.ac.uk/publications-lectures-and-talks-amtb/amt-b-4">Intelligent machinery, a heretical theory</a> (Speech). Lecture given to '51 Society'. Manchester: The Turing Digital Archive. <a rel="nofollow" class="external text" href="https://web.archive.org/web/20220926004549/https://turingarchive.kings.cam.ac.uk/publications-lectures-and-talks-amtb/amt-b-4">Archived</a> from the original on September 26, 2022. Retrieved July 22, 2022.</cite>

</li>

<li id="cite_note-89"><a href="#cite_ref-89">^</a> <cite id="CITEREFTuring1951" class="citation episode cs1">Turing, Alan (May 15, 1951). "Can digital computers think?". Automatic Calculating Machines. Episode 2. BBC. <a rel="nofollow" class="external text" href="https://turingarchive.kings.cam.ac.uk/publications-lectures-and-talks-amtb/amt-b-6">Can digital computers think?</a>.</cite>

</li>

<li id="cite_note-:3022-91"><a href="#cite_ref-:3022_91-0">^</a> <cite id="CITEREFMuehlhauser2016" class="citation web cs1">Muehlhauser, Luke (January 29, 2016). <a rel="nofollow" class="external text" href="https://lukemuehlhauser.com/sutskever-on-talking-machines/">"Sutskever on Talking Machines"</a>. Luke Muehlhauser. <a rel="nofollow" class="external text" href="https://web.archive.org/web/20220927200137/https://lukemuehlhauser.com/sutskever-on-talking-machines/">Archived</a> from the original on September 27, 2022. Retrieved August 26, 2022.</cite>

</li>

<li id="cite_note-:3122-93"><a href="#cite_ref-:3122_93-0">^</a> <cite id="CITEREFShanahan2015" class="citation book cs1">Shanahan, Murray (2015). The technological singularity. Cambridge, Massachusetts: MIT Press. <a href="ISBN_(identifier)" class="mw-redirect" title="ISBN (identifier)">ISBN</a> <bdi>978-0-262-52780-4</bdi>. <a href="OCLC_(identifier)" class="mw-redirect" title="OCLC (identifier)">OCLC</a> <a rel="nofollow" class="external text" href="https://search.worldcat.org/oclc/917889148">917889148</a>.</cite>

</li>

<li id="cite_note-:3322-95"><a href="#cite_ref-:3322_95-0">^</a> <cite id="CITEREFRossi" class="citation news cs1">Rossi, Francesca. <a rel="nofollow" class="external text" href="https://www.washingtonpost.com/news/in-theory/wp/2015/11/05/how-do-you-teach-a-machine-to-be-moral/">"How do you teach a machine to be moral?"</a>. The Washington Post. <a href="ISSN_(identifier)" class="mw-redirect" title="ISSN (identifier)">ISSN</a> <a rel="nofollow" class="external text" href="https://search.worldcat.org/issn/0190-8286">0190-8286</a>. <a rel="nofollow" class="external text" href="https://web.archive.org/web/20230210114137/https://www.washingtonpost.com/news/in-theory/wp/2015/11/05/how-do-you-teach-a-machine-to-be-moral/">Archived</a> from the original on February 10, 2023. Retrieved September 12, 2022.</cite>

</li>

<li id="cite_note-:3422-96"><a href="#cite_ref-:3422_96-0">^</a> <cite id="CITEREFAaronson2022" class="citation web cs1">Aaronson, Scott (June 17, 2022). <a rel="nofollow" class="external text" href="https://scottaaronson.blog/?p=6484">"OpenAI!"</a>. Shtetl-Optimized. <a rel="nofollow" class="external text" href="https://web.archive.org/web/20220827214238/https://scottaaronson.blog/?p=6484">Archived</a> from the original on August 27, 2022. Retrieved September 12, 2022.</cite>

</li>

<li id="cite_note-:3522-97"><a href="#cite_ref-:3522_97-0">^</a> <cite id="CITEREFSelman" class="citation cs2">Selman, Bart, <a rel="nofollow" class="external text" href="https://futureoflife.org/data/PDF/bart_selman.pdf">Intelligence Explosion: Science or Fiction?</a> (PDF), <a rel="nofollow" class="external text" href="https://web.archive.org/web/20220531022540/https://futureoflife.org/data/PDF/bart_selman.pdf">archived</a> (PDF) from the original on May 31, 2022, retrieved September 12, 2022</cite>

</li>

<li id="cite_note-:3622-98"><a href="#cite_ref-:3622_98-0">^</a> <cite id="CITEREFMcAllester2014" class="citation web cs1">McAllester (August 10, 2014). <a rel="nofollow" class="external text" href="https://machinethoughts.wordpress.com/2014/08/10/friendly-ai-and-the-servant-mission/">"Friendly AI and the Servant Mission"</a>. Machine Thoughts. <a rel="nofollow" class="external text" href="https://web.archive.org/web/20220928054922/https://machinethoughts.wordpress.com/2014/08/10/friendly-ai-and-the-servant-mission/">Archived</a> from the original on September 28, 2022. Retrieved September 12, 2022.</cite>

</li>

<li id="cite_note-AGISafetyLitReview-99">^ <a href="#cite_ref-AGISafetyLitReview_99-0">a</a> <a href="#cite_ref-AGISafetyLitReview_99-1">b</a> <a href="#cite_ref-AGISafetyLitReview_99-2">c</a> <a href="#cite_ref-AGISafetyLitReview_99-3">d</a> <a href="#cite_ref-AGISafetyLitReview_99-4">e</a> <cite id="CITEREFEverittLeaHutter2018" class="citation arxiv cs1">Everitt, Tom; Lea, Gary; Hutter, Marcus (May 21, 2018). "AGI Safety Literature Review". <a href="ArXiv_(identifier)" class="mw-redirect" title="ArXiv (identifier)">arXiv</a>:<a rel="nofollow" class="external text" href="https://arxiv.org/abs/1805.01109">1805.01109</a> [<a rel="nofollow" class="external text" href="https://arxiv.org/archive/cs.AI">cs.AI</a>].</cite>

</li>

<li id="cite_note-:3822-100"><a href="#cite_ref-:3822_100-0">^</a> <cite id="CITEREFShane2009" class="citation web cs1">Shane (August 31, 2009). <a rel="nofollow" class="external text" href="http://www.vetta.org/2009/08/funding-safe-agi/">"Funding safe AGI"</a>. vetta project. <a rel="nofollow" class="external text" href="https://web.archive.org/web/20221010143110/http://www.vetta.org/2009/08/funding-safe-agi/">Archived</a> from the original on October 10, 2022. Retrieved September 12, 2022.</cite>

</li>

<li id="cite_note-:3922-101"><a href="#cite_ref-:3922_101-0">^</a> <cite id="CITEREFHorvitz2016" class="citation web cs1">Horvitz, Eric (June 27, 2016). <a rel="nofollow" class="external text" href="http://erichorvitz.com/OSTP-CMU_AI_Safety_framing_talk.pdf">"Reflections on Safety and Artificial Intelligence"</a> (PDF). Eric Horvitz. <a rel="nofollow" class="external text" href="https://web.archive.org/web/20221010143106/http://erichorvitz.com/OSTP-CMU_AI_Safety_framing_talk.pdf">Archived</a> (PDF) from the original on October 10, 2022. Retrieved April 20, 2020.</cite>

</li>

<li id="cite_note-:4022-102"><a href="#cite_ref-:4022_102-0">^</a> <cite id="CITEREFChollet2018" class="citation web cs1">Chollet, François (December 8, 2018). <a rel="nofollow" class="external text" href="https://medium.com/@francois.chollet/the-impossibility-of-intelligence-explosion-5be4a9eda6ec">"The implausibility of intelligence explosion"</a>. Medium. <a rel="nofollow" class="external text" href="https://web.archive.org/web/20210322214203/https://medium.com/@francois.chollet/the-impossibility-of-intelligence-explosion-5be4a9eda6ec">Archived</a> from the original on March 22, 2021. Retrieved August 26, 2022.</cite>

</li>

<li id="cite_note-:4122-103"><a href="#cite_ref-:4122_103-0">^</a> <cite id="CITEREFMarcus2022" class="citation web cs1">Marcus, Gary (June 6, 2022). <a rel="nofollow" class="external text" href="https://www.scientificamerican.com/article/artificial-general-intelligence-is-not-as-imminent-as-you-might-think1/">"Artificial General Intelligence Is Not as Imminent as You Might Think"</a>. Scientific American. <a rel="nofollow" class="external text" href="https://web.archive.org/web/20220915154158/https://www.scientificamerican.com/article/artificial-general-intelligence-is-not-as-imminent-as-you-might-think1/">Archived</a> from the original on September 15, 2022. Retrieved August 26, 2022.</cite>

</li>

<li id="cite_note-:4322-104"><a href="#cite_ref-:4322_104-0">^</a> <cite id="CITEREFBarber2016" class="citation web cs1">Barber, Lynsey (July 31, 2016). <a rel="nofollow" class="external text" href="https://www.cityam.com/phew-facebooks-ai-chief-says-intelligent-machines-not/">"Phew! Facebook's AI chief says intelligent machines are not a threat to humanity"</a>. CityAM. <a rel="nofollow" class="external text" href="https://web.archive.org/web/20220826063808/https://www.cityam.com/phew-facebooks-ai-chief-says-intelligent-machines-not/">Archived</a> from the original on August 26, 2022. Retrieved August 26, 2022.</cite>

</li>

<li id="cite_note-105"><a href="#cite_ref-105">^</a> <cite id="CITEREFEtzioni2016" class="citation web cs1">Etzioni, Oren (September 20, 2016). <a rel="nofollow" class="external text" href="https://www.technologyreview.com/2016/09/20/70131/no-the-experts-dont-think-superintelligent-ai-is-a-threat-to-humanity/">"No, the Experts Don't Think Superintelligent AI is a Threat to Humanity"</a>. MIT Technology Review. Retrieved June 10, 2024.</cite>

</li>

<li id="cite_note-106"><a href="#cite_ref-106">^</a> <cite id="CITEREFRochonRossi2015" class="citation book cs1">Rochon, Louis-Philippe; Rossi, Sergio (February 27, 2015). <a rel="nofollow" class="external text" href="https://books.google.com/books?id=6kzfBgAAQBAJ">The Encyclopedia of Central Banking</a>. Edward Elgar Publishing. <a href="ISBN_(identifier)" class="mw-redirect" title="ISBN (identifier)">ISBN</a> <bdi>978-1-78254-744-0</bdi>. <a rel="nofollow" class="external text" href="https://web.archive.org/web/20230210114225/https://books.google.com/books?id=6kzfBgAAQBAJ">Archived</a> from the original on February 10, 2023. Retrieved September 13, 2022.</cite>

</li>

<li id="cite_note-107"><a href="#cite_ref-107">^</a> <cite id="CITEREFNgRussell2000" class="citation journal cs1">Ng, Andrew Y.; Russell, Stuart J. (June 29, 2000). <a rel="nofollow" class="external text" href="https://dl.acm.org/doi/10.5555/645529.657801">"Algorithms for Inverse Reinforcement Learning"</a>. Proceedings of the Seventeenth International Conference on Machine Learning. ICML '00. San Francisco, CA, USA: Morgan Kaufmann Publishers Inc.: 663–670. <a href="ISBN_(identifier)" class="mw-redirect" title="ISBN (identifier)">ISBN</a> <bdi>978-1-55860-707-1</bdi>.</cite>

</li>

<li id="cite_note-108"><a href="#cite_ref-108">^</a> <cite id="CITEREFHadfield-MenellRussellAbbeelDragan2016" class="citation conference cs1">Hadfield-Menell, Dylan; Russell, Stuart J; Abbeel, Pieter; Dragan, Anca (2016). "Cooperative inverse reinforcement learning". Advances in neural information processing systems. Vol. 29. Curran Associates, Inc.</cite>

</li>

<li id="cite_note-109"><a href="#cite_ref-109">^</a> <cite id="CITEREFMindermannArmstrong2018" class="citation conference cs1">Mindermann, Soren; Armstrong, Stuart (2018). "Occam's razor is insufficient to infer the preferences of irrational agents". Proceedings of the 32nd international conference on neural information processing systems. NIPS'18. Red Hook, NY, USA: Curran Associates Inc. pp. 5603–5614.</cite>

</li>

<li id="cite_note-110"><a href="#cite_ref-110">^</a> <cite id="CITEREFFürnkranzHüllermeierRudinSlowinski2014" class="citation journal cs1">Fürnkranz, Johannes; Hüllermeier, Eyke; Rudin, Cynthia; Slowinski, Roman; Sanner, Scott (2014). <a rel="nofollow" class="external text" href="http://drops.dagstuhl.de/opus/volltexte/2014/4550/">"Preference Learning"</a>. Dagstuhl Reports. 4 (3). Marc Herbstritt: 27 pages. <a href="Doi_(identifier)" class="mw-redirect" title="Doi (identifier)">doi</a>:<a rel="nofollow" class="external text" href="https://doi.org/10.4230%2FDAGREP.4.3.1">10.4230/DAGREP.4.3.1</a>. <a rel="nofollow" class="external text" href="https://web.archive.org/web/20230210114221/https://drops.dagstuhl.de/opus/volltexte/2014/4550/">Archived</a> from the original on February 10, 2023. Retrieved September 12, 2022.</cite>

</li>

<li id="cite_note-111"><a href="#cite_ref-111">^</a> <cite id="CITEREFGaoSchulmanHilton2022" class="citation arxiv cs1">Gao, Leo; Schulman, John; Hilton, Jacob (October 19, 2022). "Scaling Laws for Reward Model Overoptimization". <a href="ArXiv_(identifier)" class="mw-redirect" title="ArXiv (identifier)">arXiv</a>:<a rel="nofollow" class="external text" href="https://arxiv.org/abs/2210.10760">2210.10760</a> [<a rel="nofollow" class="external text" href="https://arxiv.org/archive/cs.LG">cs.LG</a>].</cite>

</li>

<li id="cite_note-112"><a href="#cite_ref-112">^</a> <cite id="CITEREFAnderson2022" class="citation web cs1">Anderson, Martin (April 5, 2022). <a rel="nofollow" class="external text" href="https://www.unite.ai/the-perils-of-using-quotations-to-authenticate-nlg-content/">"The Perils of Using Quotations to Authenticate NLG Content"</a>. Unite.AI. <a rel="nofollow" class="external text" href="https://web.archive.org/web/20230210114139/https://www.unite.ai/the-perils-of-using-quotations-to-authenticate-nlg-content/">Archived</a> from the original on February 10, 2023. Retrieved July 21, 2022.</cite>

</li>

<li id="cite_note-Wiggers2022-113">^ <a href="#cite_ref-Wiggers2022_113-0">a</a> <a href="#cite_ref-Wiggers2022_113-1">b</a> <cite id="CITEREFWiggers2022" class="citation web cs1">Wiggers, Kyle (February 5, 2022). <a rel="nofollow" class="external text" href="https://venturebeat.com/2022/02/05/despite-recent-progress-ai-powered-chatbots-still-have-a-long-way-to-go/">"Despite recent progress, AI-powered chatbots still have a long way to go"</a>. VentureBeat. <a rel="nofollow" class="external text" href="https://web.archive.org/web/20220723184144/https://venturebeat.com/2022/02/05/despite-recent-progress-ai-powered-chatbots-still-have-a-long-way-to-go/">Archived</a> from the original on July 23, 2022. Retrieved July 23, 2022.</cite>

</li>

<li id="cite_note-114"><a href="#cite_ref-114">^</a> <cite id="CITEREFHendrycksBurnsBasartCritch2021" class="citation journal cs1">Hendrycks, Dan; Burns, Collin; Basart, Steven; Critch, Andrew; Li, Jerry; Song, Dawn; Steinhardt, Jacob (July 24, 2021). "Aligning AI With Shared Human Values". International Conference on Learning Representations. <a href="ArXiv_(identifier)" class="mw-redirect" title="ArXiv (identifier)">arXiv</a>:<a rel="nofollow" class="external text" href="https://arxiv.org/abs/2008.02275">2008.02275</a>.</cite>

</li>

<li id="cite_note-115"><a href="#cite_ref-115">^</a> <cite id="CITEREFPerezHuangSongCai2022" class="citation arxiv cs1">Perez, Ethan; Huang, Saffron; Song, Francis; Cai, Trevor; Ring, Roman; Aslanides, John; Glaese, Amelia; McAleese, Nat; Irving, Geoffrey (February 7, 2022). "Red Teaming Language Models with Language Models". <a href="ArXiv_(identifier)" class="mw-redirect" title="ArXiv (identifier)">arXiv</a>:<a rel="nofollow" class="external text" href="https://arxiv.org/abs/2202.03286">2202.03286</a> [<a rel="nofollow" class="external text" href="https://arxiv.org/archive/cs.CL">cs.CL</a>].</cite>

<ul><li><cite id="CITEREFBhattacharyya2022" class="citation web cs1">Bhattacharyya, Sreejani (February 14, 2022). <a rel="nofollow" class="external text" href="https://analyticsindiamag.com/deepminds-red-teaming-language-models-with-language-models-what-is-it/">"DeepMind's "red teaming" language models with language models: What is it?"</a>. Analytics India Magazine. <a rel="nofollow" class="external text" href="https://web.archive.org/web/20230213145212/https://analyticsindiamag.com/deepminds-red-teaming-language-models-with-language-models-what-is-it/">Archived</a> from the original on February 13, 2023. Retrieved July 23, 2022.</cite></li></ul>

</li>

<li id="cite_note-116"><a href="#cite_ref-116">^</a> <cite id="CITEREFAndersonAnderson2007" class="citation journal cs1">Anderson, Michael; Anderson, Susan Leigh (December 15, 2007). <a rel="nofollow" class="external text" href="https://ojs.aaai.org/aimagazine/index.php/aimagazine/article/view/2065">"Machine Ethics: Creating an Ethical Intelligent Agent"</a>. AI Magazine. 28 (4): 15. <a href="Doi_(identifier)" class="mw-redirect" title="Doi (identifier)">doi</a>:<a rel="nofollow" class="external text" href="https://doi.org/10.1609%2Faimag.v28i4.2065">10.1609/aimag.v28i4.2065</a>. <a href="ISSN_(identifier)" class="mw-redirect" title="ISSN (identifier)">ISSN</a> <a rel="nofollow" class="external text" href="https://search.worldcat.org/issn/2371-9621">2371-9621</a>. <a href="S2CID_(identifier)" class="mw-redirect" title="S2CID (identifier)">S2CID</a> <a rel="nofollow" class="external text" href="https://api.semanticscholar.org/CorpusID:17033332">17033332</a>. Retrieved March 14, 2023.</cite>

</li>

<li id="cite_note-117"><a href="#cite_ref-117">^</a> <cite id="CITEREFWiegel2010" class="citation journal cs1">Wiegel, Vincent (December 1, 2010). <a rel="nofollow" class="external text" href="https://doi.org/10.1007%2Fs10676-010-9239-1">"Wendell Wallach and Colin Allen: moral machines: teaching robots right from wrong"</a>. Ethics and Information Technology. 12 (4): 359–361. <a href="Doi_(identifier)" class="mw-redirect" title="Doi (identifier)">doi</a>:<a rel="nofollow" class="external text" href="https://doi.org/10.1007%2Fs10676-010-9239-1">10.1007/s10676-010-9239-1</a>. <a href="ISSN_(identifier)" class="mw-redirect" title="ISSN (identifier)">ISSN</a> <a rel="nofollow" class="external text" href="https://search.worldcat.org/issn/1572-8439">1572-8439</a>. <a href="S2CID_(identifier)" class="mw-redirect" title="S2CID (identifier)">S2CID</a> <a rel="nofollow" class="external text" href="https://api.semanticscholar.org/CorpusID:30532107">30532107</a>.</cite>

</li>

<li id="cite_note-118"><a href="#cite_ref-118">^</a> <cite id="CITEREFWallachAllen2009" class="citation book cs1">Wallach, Wendell; Allen, Colin (2009). <a rel="nofollow" class="external text" href="https://oxford.universitypressscholarship.com/10.1093/acprof:oso/9780195374049.001.0001/acprof-9780195374049">Moral Machines: Teaching Robots Right from Wrong</a>. New York: Oxford University Press. <a href="ISBN_(identifier)" class="mw-redirect" title="ISBN (identifier)">ISBN</a> <bdi>978-0-19-537404-9</bdi>. <a rel="nofollow" class="external text" href="https://web.archive.org/web/20230315193012/https://academic.oup.com/pages/op-migration-welcome">Archived</a> from the original on March 15, 2023.</cite>

</li>

<li id="cite_note-Phelps2023-120">^ <a href="#cite_ref-Phelps2023_120-0">a</a> <a href="#cite_ref-Phelps2023_120-1">b</a> <cite id="CITEREFPhelpsRanson2023" class="citation arxiv cs1">Phelps, Steve; Ranson, Rebecca (2023). "Of Models and Tin-Men - A Behavioral Economics Study of Principal-Agent Problems in AI Alignment Using Large-Language Models". <a href="ArXiv_(identifier)" class="mw-redirect" title="ArXiv (identifier)">arXiv</a>:<a rel="nofollow" class="external text" href="https://arxiv.org/abs/2307.11137">2307.11137</a> [<a rel="nofollow" class="external text" href="https://arxiv.org/archive/cs.AI">cs.AI</a>].</cite>

</li>

<li id="cite_note-121"><a href="#cite_ref-121">^</a> <cite id="CITEREFHendrycksBurnsBasartCritch2020" class="citation arxiv cs1">Hendrycks, Dan; Burns, Collin; Basart, Steven; Critch, Andrew; Li, Jerry; Song, Dawn; Steinhardt, Jacob (2020). "Aligning AI With Shared Human Values". <a href="ArXiv_(identifier)" class="mw-redirect" title="ArXiv (identifier)">arXiv</a>:<a rel="nofollow" class="external text" href="https://arxiv.org/abs/2008.02275">2008.02275</a> [<a rel="nofollow" class="external text" href="https://arxiv.org/archive/cs.CY">cs.CY</a>].</cite>

</li>

<li id="cite_note-122"><a href="#cite_ref-122">^</a> <cite id="CITEREFMacAskill2022" class="citation book cs1">MacAskill, William (2022). <a rel="nofollow" class="external text" href="https://whatweowethefuture.com/">What we owe the future</a>. New York, NY: Basic Books, Hachette Book Group. <a href="ISBN_(identifier)" class="mw-redirect" title="ISBN (identifier)">ISBN</a> <bdi>978-1-5416-1862-6</bdi>. <a href="OCLC_(identifier)" class="mw-redirect" title="OCLC (identifier)">OCLC</a> <a rel="nofollow" class="external text" href="https://search.worldcat.org/oclc/1314633519">1314633519</a>. <a rel="nofollow" class="external text" href="https://web.archive.org/web/20220914030758/https://www.basicbooks.com/titles/william-macaskill/what-we-owe-the-future/9781541618633/">Archived</a> from the original on September 14, 2022. Retrieved September 11, 2024.</cite>

</li>

<li id="cite_note-RecursivelySummarizing-123">^ <a href="#cite_ref-RecursivelySummarizing_123-0">a</a> <a href="#cite_ref-RecursivelySummarizing_123-1">b</a> <cite id="CITEREFWuOuyangZieglerStiennon2021" class="citation arxiv cs1">Wu, Jeff; Ouyang, Long; Ziegler, Daniel M.; Stiennon, Nisan; Lowe, Ryan; Leike, Jan; Christiano, Paul (September 27, 2021). "Recursively Summarizing Books with Human Feedback". <a href="ArXiv_(identifier)" class="mw-redirect" title="ArXiv (identifier)">arXiv</a>:<a rel="nofollow" class="external text" href="https://arxiv.org/abs/2109.10862">2109.10862</a> [<a rel="nofollow" class="external text" href="https://arxiv.org/archive/cs.CL">cs.CL</a>].</cite>

</li>

<li id="cite_note-124"><a href="#cite_ref-124">^</a> <cite id="CITEREFPearceAhmadTanDolan-Gavitt2022" class="citation book cs1">Pearce, Hammond; Ahmad, Baleegh; Tan, Benjamin; Dolan-Gavitt, Brendan; Karri, Ramesh (2022). "Asleep at the Keyboard? Assessing the Security of GitHub Copilot's Code Contributions". 2022 IEEE Symposium on Security and Privacy (SP). San Francisco, CA, USA: IEEE. pp. 754–768. <a href="ArXiv_(identifier)" class="mw-redirect" title="ArXiv (identifier)">arXiv</a>:<a rel="nofollow" class="external text" href="https://arxiv.org/abs/2108.09293">2108.09293</a>. <a href="Doi_(identifier)" class="mw-redirect" title="Doi (identifier)">doi</a>:<a rel="nofollow" class="external text" href="https://doi.org/10.1109%2FSP46214.2022.9833571">10.1109/SP46214.2022.9833571</a>. <a href="ISBN_(identifier)" class="mw-redirect" title="ISBN (identifier)">ISBN</a> <bdi>978-1-6654-1316-9</bdi>. <a href="S2CID_(identifier)" class="mw-redirect" title="S2CID (identifier)">S2CID</a> <a rel="nofollow" class="external text" href="https://api.semanticscholar.org/CorpusID:245220588">245220588</a>.</cite>

</li>

<li id="cite_note-125"><a href="#cite_ref-125">^</a> <cite id="CITEREFIrvingAmodei2018" class="citation web cs1">Irving, Geoffrey; Amodei, Dario (May 3, 2018). <a rel="nofollow" class="external text" href="https://openai.com/blog/debate/">"AI Safety via Debate"</a>. OpenAI. <a rel="nofollow" class="external text" href="https://web.archive.org/web/20230210114137/https://openai.com/blog/debate/">Archived</a> from the original on February 10, 2023. Retrieved July 23, 2022.</cite>

</li>

<li id="cite_note-sslawe-126">^ <a href="#cite_ref-sslawe_126-0">a</a> <a href="#cite_ref-sslawe_126-1">b</a> <cite id="CITEREFChristianoShlegerisAmodei2018" class="citation arxiv cs1">Christiano, Paul; Shlegeris, Buck; Amodei, Dario (October 19, 2018). "Supervising strong learners by amplifying weak experts". <a href="ArXiv_(identifier)" class="mw-redirect" title="ArXiv (identifier)">arXiv</a>:<a rel="nofollow" class="external text" href="https://arxiv.org/abs/1810.08575">1810.08575</a> [<a rel="nofollow" class="external text" href="https://arxiv.org/archive/cs.LG">cs.LG</a>].</cite>

</li>

<li id="cite_note-127"><a href="#cite_ref-127">^</a> <cite id="CITEREFBanzhafGoodmanShenemanTrujillo2020" class="citation book cs1">Banzhaf, Wolfgang; Goodman, Erik; Sheneman, Leigh; Trujillo, Leonardo; Worzel, Bill, eds. (2020). <a rel="nofollow" class="external text" href="http://link.springer.com/10.1007/978-3-030-39958-0">Genetic Programming Theory and Practice XVII</a>. Genetic and Evolutionary Computation. Cham: Springer International Publishing. <a href="Doi_(identifier)" class="mw-redirect" title="Doi (identifier)">doi</a>:<a rel="nofollow" class="external text" href="https://doi.org/10.1007%2F978-3-030-39958-0">10.1007/978-3-030-39958-0</a>. <a href="ISBN_(identifier)" class="mw-redirect" title="ISBN (identifier)">ISBN</a> <bdi>978-3-030-39957-3</bdi>. <a href="S2CID_(identifier)" class="mw-redirect" title="S2CID (identifier)">S2CID</a> <a rel="nofollow" class="external text" href="https://api.semanticscholar.org/CorpusID:218531292">218531292</a>. <a rel="nofollow" class="external text" href="https://web.archive.org/web/20230315193000/https://link.springer.com/book/10.1007/978-3-030-39958-0">Archived</a> from the original on March 15, 2023.</cite>

</li>

<li id="cite_note-128"><a href="#cite_ref-128">^</a> <cite id="CITEREFWiblin2018" class="citation podcast cs1">Wiblin, Robert (October 2, 2018). <a rel="nofollow" class="external text" href="https://80000hours.org/podcast/episodes/paul-christiano-ai-alignment-solutions/">"Dr Paul Christiano on how OpenAI is developing real solutions to the 'AI alignment problem', and his vision of how humanity will progressively hand over decision-making to AI systems"</a> (Podcast). 80,000 hours. No. 44. <a rel="nofollow" class="external text" href="https://web.archive.org/web/20221214050326/https://80000hours.org/podcast/episodes/paul-christiano-ai-alignment-solutions/">Archived</a> from the original on December 14, 2022. Retrieved July 23, 2022.</cite>

</li>

<li id="cite_note-129"><a href="#cite_ref-129">^</a> <cite id="CITEREFLehmanCluneMisevicAdami2020" class="citation journal cs1">Lehman, Joel; Clune, Jeff; Misevic, Dusan; Adami, Christoph; Altenberg, Lee; Beaulieu, Julie; Bentley, Peter J.; Bernard, Samuel; Beslon, Guillaume; Bryson, David M.; Cheney, Nick (2020). <a rel="nofollow" class="external text" href="https://direct.mit.edu/artl/article/26/2/274-306/93255">"The Surprising Creativity of Digital Evolution: A Collection of Anecdotes from the Evolutionary Computation and Artificial Life Research Communities"</a>. Artificial Life. 26 (2): 274–306. <a href="Doi_(identifier)" class="mw-redirect" title="Doi (identifier)">doi</a>:<a rel="nofollow" class="external text" href="https://doi.org/10.1162%2Fartl_a_00319">10.1162/artl_a_00319</a>. <a href="Hdl_(identifier)" class="mw-redirect" title="Hdl (identifier)">hdl</a>:<a rel="nofollow" class="external text" href="https://hdl.handle.net/10044%2F1%2F83343">10044/1/83343</a>. <a href="ISSN_(identifier)" class="mw-redirect" title="ISSN (identifier)">ISSN</a> <a rel="nofollow" class="external text" href="https://search.worldcat.org/issn/1064-5462">1064-5462</a>. <a href="PMID_(identifier)" class="mw-redirect" title="PMID (identifier)">PMID</a> <a rel="nofollow" class="external text" href="https://pubmed.ncbi.nlm.nih.gov/32271631">32271631</a>. <a href="S2CID_(identifier)" class="mw-redirect" title="S2CID (identifier)">S2CID</a> <a rel="nofollow" class="external text" href="https://api.semanticscholar.org/CorpusID:4519185">4519185</a>. <a rel="nofollow" class="external text" href="https://web.archive.org/web/20221010143108/https://direct.mit.edu/artl/article/26/2/274-306/93255">Archived</a> from the original on October 10, 2022. Retrieved September 12, 2022.</cite>

</li>

<li id="cite_note-saavrm-130">^ <a href="#cite_ref-saavrm_130-0">a</a> <a href="#cite_ref-saavrm_130-1">b</a> <cite id="CITEREFLeikeKruegerEverittMartic2018" class="citation arxiv cs1">Leike, Jan; Krueger, David; Everitt, Tom; Martic, Miljan; Maini, Vishal; Legg, Shane (November 19, 2018). "Scalable agent alignment via reward modeling: a research direction". <a href="ArXiv_(identifier)" class="mw-redirect" title="ArXiv (identifier)">arXiv</a>:<a rel="nofollow" class="external text" href="https://arxiv.org/abs/1811.07871">1811.07871</a> [<a rel="nofollow" class="external text" href="https://arxiv.org/archive/cs.LG">cs.LG</a>].</cite>

</li>

<li id="cite_note-OpenAIApproach-131">^ <a href="#cite_ref-OpenAIApproach_131-0">a</a> <a href="#cite_ref-OpenAIApproach_131-1">b</a> <cite id="CITEREFLeikeSchulmanWu2022" class="citation web cs1">Leike, Jan; Schulman, John; Wu, Jeffrey (August 24, 2022). <a rel="nofollow" class="external text" href="https://openai.com/blog/our-approach-to-alignment-research/">"Our approach to alignment research"</a>. OpenAI. <a rel="nofollow" class="external text" href="https://web.archive.org/web/20230215193559/https://openai.com/blog/our-approach-to-alignment-research/">Archived</a> from the original on February 15, 2023. Retrieved September 9, 2022.</cite>

</li>

<li id="cite_note-132"><a href="#cite_ref-132">^</a> <cite id="CITEREFWiggers2021" class="citation web cs1">Wiggers, Kyle (September 23, 2021). <a rel="nofollow" class="external text" href="https://venturebeat.com/2021/09/23/openai-unveils-model-that-can-summarize-books-of-any-length/">"OpenAI unveils model that can summarize books of any length"</a>. VentureBeat. <a rel="nofollow" class="external text" href="https://web.archive.org/web/20220723215104/https://venturebeat.com/2021/09/23/openai-unveils-model-that-can-summarize-books-of-any-length/">Archived</a> from the original on July 23, 2022. Retrieved July 23, 2022.</cite>

</li>

<li id="cite_note-133"><a href="#cite_ref-133">^</a> <cite id="CITEREFSaundersYehWuBills2022" class="citation arxiv cs1">Saunders, William; Yeh, Catherine; Wu, Jeff; Bills, Steven; Ouyang, Long; Ward, Jonathan; Leike, Jan (June 13, 2022). "Self-critiquing models for assisting human evaluators". <a href="ArXiv_(identifier)" class="mw-redirect" title="ArXiv (identifier)">arXiv</a>:<a rel="nofollow" class="external text" href="https://arxiv.org/abs/2206.05802">2206.05802</a> [<a rel="nofollow" class="external text" href="https://arxiv.org/archive/cs.CL">cs.CL</a>].</cite>

<ul><li><cite id="CITEREFBaiKadavathKunduAskell2022" class="citation arxiv cs1">Bai, Yuntao; Kadavath, Saurav; Kundu, Sandipan; Askell, Amanda; Kernion, Jackson; Jones, Andy; Chen, Anna; Goldie, Anna; Mirhoseini, Azalia; McKinnon, Cameron; Chen, Carol; Olsson, Catherine; Olah, Christopher; Hernandez, Danny; Drain, Dawn (December 15, 2022). "Constitutional AI: Harmlessness from AI Feedback". <a href="ArXiv_(identifier)" class="mw-redirect" title="ArXiv (identifier)">arXiv</a>:<a rel="nofollow" class="external text" href="https://arxiv.org/abs/2212.08073">2212.08073</a> [<a rel="nofollow" class="external text" href="https://arxiv.org/archive/cs.CL">cs.CL</a>].</cite></li></ul>

</li>

<li id="cite_note-134"><a href="#cite_ref-134">^</a> <cite class="citation web cs1"><a rel="nofollow" class="external text" href="https://openai.com/blog/introducing-superalignment">"Introducing Superalignment"</a>. openai.com. Retrieved July 17, 2023.</cite>

</li>

<li id="cite_note-Falsehoods-135">^ <a href="#cite_ref-Falsehoods_135-0">a</a> <a href="#cite_ref-Falsehoods_135-1">b</a> <a href="#cite_ref-Falsehoods_135-2">c</a> <cite id="CITEREFWiggers2021" class="citation web cs1">Wiggers, Kyle (September 20, 2021). <a rel="nofollow" class="external text" href="https://venturebeat.com/2021/09/20/falsehoods-more-likely-with-large-language-models/">"Falsehoods more likely with large language models"</a>. VentureBeat. <a rel="nofollow" class="external text" href="https://web.archive.org/web/20220804142703/https://venturebeat.com/2021/09/20/falsehoods-more-likely-with-large-language-models/">Archived</a> from the original on August 4, 2022. Retrieved July 23, 2022.</cite>

</li>

<li id="cite_note-136"><a href="#cite_ref-136">^</a> <cite id="CITEREFThe_Guardian2020" class="citation news cs1">The Guardian (September 8, 2020). <a rel="nofollow" class="external text" href="https://www.theguardian.com/commentisfree/2020/sep/08/robot-wrote-this-article-gpt-3">"A robot wrote this entire article. Are you scared yet, human?"</a>. The Guardian. <a href="ISSN_(identifier)" class="mw-redirect" title="ISSN (identifier)">ISSN</a> <a rel="nofollow" class="external text" href="https://search.worldcat.org/issn/0261-3077">0261-3077</a>. <a rel="nofollow" class="external text" href="https://web.archive.org/web/20200908090812/https://www.theguardian.com/commentisfree/2020/sep/08/robot-wrote-this-article-gpt-3">Archived</a> from the original on September 8, 2020. Retrieved July 23, 2022.</cite>

<ul><li><cite id="CITEREFHeaven2020" class="citation web cs1">Heaven, Will Douglas (July 20, 2020). <a rel="nofollow" class="external text" href="https://www.technologyreview.com/2020/07/20/1005454/openai-machine-learning-language-generator-gpt-3-nlp/">"OpenAI's new language generator GPT-3 is shockingly good—and completely mindless"</a>. MIT Technology Review. <a rel="nofollow" class="external text" href="https://web.archive.org/web/20200725175436/https://www.technologyreview.com/2020/07/20/1005454/openai-machine-learning-language-generator-gpt-3-nlp/">Archived</a> from the original on July 25, 2020. Retrieved July 23, 2022.</cite></li></ul>

</li>

<li id="cite_note-TruthfulAI-137">^ <a href="#cite_ref-TruthfulAI_137-0">a</a> <a href="#cite_ref-TruthfulAI_137-1">b</a> <cite id="CITEREFEvansCotton-BarrattFinnvedenBales2021" class="citation arxiv cs1">Evans, Owain; Cotton-Barratt, Owen; Finnveden, Lukas; Bales, Adam; Balwit, Avital; Wills, Peter; Righetti, Luca; Saunders, William (October 13, 2021). "Truthful AI: Developing and governing AI that does not lie". <a href="ArXiv_(identifier)" class="mw-redirect" title="ArXiv (identifier)">arXiv</a>:<a rel="nofollow" class="external text" href="https://arxiv.org/abs/2110.06674">2110.06674</a> [<a rel="nofollow" class="external text" href="https://arxiv.org/archive/cs.CY">cs.CY</a>].</cite>

</li>

<li id="cite_note-138"><a href="#cite_ref-138">^</a> <cite id="CITEREFAlford2021" class="citation web cs1">Alford, Anthony (July 13, 2021). <a rel="nofollow" class="external text" href="https://www.infoq.com/news/2021/07/eleutherai-gpt-j/">"EleutherAI Open-Sources Six Billion Parameter GPT-3 Clone GPT-J"</a>. InfoQ. <a rel="nofollow" class="external text" href="https://web.archive.org/web/20230210114137/https://www.infoq.com/news/2021/07/eleutherai-gpt-j/">Archived</a> from the original on February 10, 2023. Retrieved July 23, 2022.</cite>

<ul><li><cite id="CITEREFRaeBorgeaudCaiMillican2022" class="citation arxiv cs1">Rae, Jack W.; Borgeaud, Sebastian; Cai, Trevor; Millican, Katie; Hoffmann, Jordan; Song, Francis; Aslanides, John; Henderson, Sarah; Ring, Roman; Young, Susannah; Rutherford, Eliza; Hennigan, Tom; Menick, Jacob; Cassirer, Albin; Powell, Richard (January 21, 2022). "Scaling Language Models: Methods, Analysis & Insights from Training Gopher". <a href="ArXiv_(identifier)" class="mw-redirect" title="ArXiv (identifier)">arXiv</a>:<a rel="nofollow" class="external text" href="https://arxiv.org/abs/2112.11446">2112.11446</a> [<a rel="nofollow" class="external text" href="https://arxiv.org/archive/cs.CL">cs.CL</a>].</cite></li></ul>

</li>

<li id="cite_note-139"><a href="#cite_ref-139">^</a> <cite id="CITEREFNakanoHiltonBalajiWu2022" class="citation arxiv cs1">Nakano, Reiichiro; Hilton, Jacob; <a href="Suchir_Balaji" title="Suchir Balaji">Balaji, Suchir</a>; Wu, Jeff; Ouyang, Long; Kim, Christina; Hesse, Christopher; Jain, Shantanu; Kosaraju, Vineet; Saunders, William; Jiang, Xu; Cobbe, Karl; Eloundou, Tyna; Krueger, Gretchen; Button, Kevin (June 1, 2022). "WebGPT: Browser-assisted question-answering with human feedback". <a href="ArXiv_(identifier)" class="mw-redirect" title="ArXiv (identifier)">arXiv</a>:<a rel="nofollow" class="external text" href="https://arxiv.org/abs/2112.09332">2112.09332</a> [<a rel="nofollow" class="external text" href="https://arxiv.org/archive/cs.CL">cs.CL</a>].</cite>

<ul><li><cite id="CITEREFKumar2021" class="citation web cs1">Kumar, Nitish (December 23, 2021). <a rel="nofollow" class="external text" href="https://www.marktechpost.com/2021/12/22/openai-researchers-find-ways-to-more-accurately-answer-open-ended-questions-using-a-text-based-web-browser/">"OpenAI Researchers Find Ways To More Accurately Answer Open-Ended Questions Using A Text-Based Web Browser"</a>. MarkTechPost. <a rel="nofollow" class="external text" href="https://web.archive.org/web/20230210114137/https://www.marktechpost.com/2021/12/22/openai-researchers-find-ways-to-more-accurately-answer-open-ended-questions-using-a-text-based-web-browser/">Archived</a> from the original on February 10, 2023. Retrieved July 23, 2022.</cite></li>

<li><cite id="CITEREFMenickTrebaczMikulikAslanides2022" class="citation journal cs1">Menick, Jacob; Trebacz, Maja; Mikulik, Vladimir; Aslanides, John; Song, Francis; Chadwick, Martin; Glaese, Mia; Young, Susannah; Campbell-Gillingham, Lucy; Irving, Geoffrey; McAleese, Nat (March 21, 2022). <a rel="nofollow" class="external text" href="https://www.deepmind.com/publications/gophercite-teaching-language-models-to-support-answers-with-verified-quotes">"Teaching language models to support answers with verified quotes"</a>. DeepMind. <a href="ArXiv_(identifier)" class="mw-redirect" title="ArXiv (identifier)">arXiv</a>:<a rel="nofollow" class="external text" href="https://arxiv.org/abs/2203.11147">2203.11147</a>. <a rel="nofollow" class="external text" href="https://web.archive.org/web/20230210114137/https://www.deepmind.com/publications/gophercite-teaching-language-models-to-support-answers-with-verified-quotes">Archived</a> from the original on February 10, 2023. Retrieved September 12, 2022.</cite></li></ul>

</li>

<li id="cite_note-140"><a href="#cite_ref-140">^</a> <cite id="CITEREFAskellBaiChenDrain2021" class="citation arxiv cs1">Askell, Amanda; Bai, Yuntao; Chen, Anna; Drain, Dawn; Ganguli, Deep; Henighan, Tom; Jones, Andy; Joseph, Nicholas; Mann, Ben; DasSarma, Nova; Elhage, Nelson; Hatfield-Dodds, Zac; Hernandez, Danny; Kernion, Jackson; Ndousse, Kamal (December 9, 2021). "A General Language Assistant as a Laboratory for Alignment". <a href="ArXiv_(identifier)" class="mw-redirect" title="ArXiv (identifier)">arXiv</a>:<a rel="nofollow" class="external text" href="https://arxiv.org/abs/2112.00861">2112.00861</a> [<a rel="nofollow" class="external text" href="https://arxiv.org/archive/cs.CL">cs.CL</a>].</cite>

</li>

<li id="cite_note-141"><a href="#cite_ref-141">^</a> <cite id="CITEREFCox2023" class="citation web cs1">Cox, Joseph (March 15, 2023). <a rel="nofollow" class="external text" href="https://www.vice.com/en/article/gpt4-hired-unwitting-taskrabbit-worker/">"GPT-4 Hired Unwitting TaskRabbit Worker By Pretending to Be 'Vision-Impaired' Human"</a>. Vice. Retrieved April 10, 2023.</cite>

</li>

<li id="cite_note-142"><a href="#cite_ref-142">^</a> <cite id="CITEREFScheurerBalesniHobbhahn2023" class="citation arxiv cs1">Scheurer, Jérémy; Balesni, Mikita; Hobbhahn, Marius (2023). "Technical Report: Large Language Models can Strategically Deceive their Users when Put Under Pressure". <a href="ArXiv_(identifier)" class="mw-redirect" title="ArXiv (identifier)">arXiv</a>:<a rel="nofollow" class="external text" href="https://arxiv.org/abs/2311.07590">2311.07590</a> [<a rel="nofollow" class="external text" href="https://arxiv.org/archive/cs.CL">cs.CL</a>].</cite>

</li>

<li id="cite_note-143"><a href="#cite_ref-143">^</a> <cite id="CITEREFKentonEverittWeidingerGabriel2021" class="citation web cs1">Kenton, Zachary; Everitt, Tom; Weidinger, Laura; Gabriel, Iason; Mikulik, Vladimir; Irving, Geoffrey (March 30, 2021). <a rel="nofollow" class="external text" href="https://deepmindsafetyresearch.medium.com/alignment-of-language-agents-9fbc7dd52c6c">"Alignment of Language Agents"</a>. DeepMind Safety Research – Medium. <a rel="nofollow" class="external text" href="https://web.archive.org/web/20230210114142/https://deepmindsafetyresearch.medium.com/alignment-of-language-agents-9fbc7dd52c6c">Archived</a> from the original on February 10, 2023. Retrieved July 23, 2022.</cite>

</li>

<li id="cite_note-144"><a href="#cite_ref-144">^</a> <cite id="CITEREFParkGoldsteinO’GaraChen2024" class="citation journal cs1">Park, Peter S.; Goldstein, Simon; O’Gara, Aidan; Chen, Michael; Hendrycks, Dan (May 2024). <a rel="nofollow" class="external text" href="https://doi.org/10.1016/j.patter.2024.100988">"AI deception: A survey of examples, risks, and potential solutions"</a>. Patterns. 5 (5): 100988. <a href="Doi_(identifier)" class="mw-redirect" title="Doi (identifier)">doi</a>:<a rel="nofollow" class="external text" href="https://doi.org/10.1016%2Fj.patter.2024.100988">10.1016/j.patter.2024.100988</a>. <a href="ISSN_(identifier)" class="mw-redirect" title="ISSN (identifier)">ISSN</a> <a rel="nofollow" class="external text" href="https://search.worldcat.org/issn/2666-3899">2666-3899</a>. <a href="PMC_(identifier)" class="mw-redirect" title="PMC (identifier)">PMC</a> <a rel="nofollow" class="external text" href="https://www.ncbi.nlm.nih.gov/pmc/articles/PMC11117051">11117051</a>. <a href="PMID_(identifier)" class="mw-redirect" title="PMID (identifier)">PMID</a> <a rel="nofollow" class="external text" href="https://pubmed.ncbi.nlm.nih.gov/38800366">38800366</a>.</cite>

</li>

<li id="cite_note-145"><a href="#cite_ref-145">^</a> <cite id="CITEREFZia2025" class="citation web cs1">Zia, Tehseen (January 7, 2025). <a rel="nofollow" class="external text" href="https://www.unite.ai/can-ai-be-trusted-the-challenge-of-alignment-faking/">"Can AI Be Trusted? The Challenge of Alignment Faking"</a>. Unite.AI. Retrieved February 18, 2025.</cite>

</li>

<li id="cite_note-:6-146">^ <a href="#cite_ref-:6_146-0">a</a> <a href="#cite_ref-:6_146-1">b</a> <cite id="CITEREFPerrigo2024" class="citation magazine cs1">Perrigo, Billy (December 18, 2024). <a rel="nofollow" class="external text" href="https://time.com/7202784/ai-research-strategic-lying/">"Exclusive: New Research Shows AI Strategically Lying"</a>. TIME. Retrieved February 18, 2025.</cite>

</li>

<li id="cite_note-147"><a href="#cite_ref-147">^</a> <cite class="citation web cs1"><a rel="nofollow" class="external text" href="https://www.anthropic.com/research/alignment-faking">"Alignment faking in large language models"</a>. Anthropic. December 18, 2024. Retrieved February 17, 2025.</cite>

</li>

<li id="cite_note-148"><a href="#cite_ref-148">^</a> <cite id="CITEREFGreenblattDenisonWrightRoger2024" class="citation arxiv cs1">Greenblatt, Ryan; Denison, Carson; Wright, Benjamin; Roger, Fabien; MacDiarmid, Monte; Marks, Sam; Treutlein, Johannes; Belonax, Tim; Chen, Jack (December 20, 2024). "Alignment faking in large language models". <a href="ArXiv_(identifier)" class="mw-redirect" title="ArXiv (identifier)">arXiv</a>:<a rel="nofollow" class="external text" href="https://arxiv.org/abs/2412.14093">2412.14093</a> [<a rel="nofollow" class="external text" href="https://arxiv.org/archive/cs.AI">cs.AI</a>].</cite>

</li>

<li id="cite_note-149"><a href="#cite_ref-149">^</a> <cite id="CITEREFMcCarthyMinskyRochesterShannon2006" class="citation journal cs1">McCarthy, John; Minsky, Marvin L.; Rochester, Nathaniel; Shannon, Claude E. (December 15, 2006). <a rel="nofollow" class="external text" href="https://ojs.aaai.org/aimagazine/index.php/aimagazine/article/view/1904">"A Proposal for the Dartmouth Summer Research Project on Artificial Intelligence, August 31, 1955"</a>. AI Magazine. 27 (4): 12. <a href="Doi_(identifier)" class="mw-redirect" title="Doi (identifier)">doi</a>:<a rel="nofollow" class="external text" href="https://doi.org/10.1609%2Faimag.v27i4.1904">10.1609/aimag.v27i4.1904</a>. <a href="ISSN_(identifier)" class="mw-redirect" title="ISSN (identifier)">ISSN</a> <a rel="nofollow" class="external text" href="https://search.worldcat.org/issn/2371-9621">2371-9621</a>. <a href="S2CID_(identifier)" class="mw-redirect" title="S2CID (identifier)">S2CID</a> <a rel="nofollow" class="external text" href="https://api.semanticscholar.org/CorpusID:19439915">19439915</a>.</cite>

</li>

<li id="cite_note-150"><a href="#cite_ref-150">^</a> <cite id="CITEREFWangMaFengZhang2024" class="citation cs2">Wang, Lei; Ma, Chen; Feng, Xueyang; Zhang, Zeyu; Yang, Hao; Zhang, Jingsen; Chen, Zhiyuan; Tang, Jiakai; Chen, Xu (2024), "A survey on large language model based autonomous agents", Frontiers of Computer Science, 18 (6) 186345, <a href="ArXiv_(identifier)" class="mw-redirect" title="ArXiv (identifier)">arXiv</a>:<a rel="nofollow" class="external text" href="https://arxiv.org/abs/2308.11432">2308.11432</a>, <a href="Doi_(identifier)" class="mw-redirect" title="Doi (identifier)">doi</a>:<a rel="nofollow" class="external text" href="https://doi.org/10.1007%2Fs11704-024-40231-1">10.1007/s11704-024-40231-1</a></cite>

</li>

<li id="cite_note-151"><a href="#cite_ref-151">^</a> <cite class="citation web cs1"><a rel="nofollow" class="external text" href="https://fortune.com/2023/05/02/godfather-ai-geoff-hinton-google-warns-artificial-intelligence-nightmare-scenario/">"'The Godfather of A.I.' warns of 'nightmare scenario' where artificial intelligence begins to seek power"</a>. Fortune. Retrieved May 4, 2023.</cite>

<ul><li><cite class="citation news cs1"><a rel="nofollow" class="external text" href="https://www.technologyreview.com/2016/11/02/156285/yes-we-are-worried-about-the-existential-risk-of-artificial-intelligence/">"Yes, We Are Worried About the Existential Risk of Artificial Intelligence"</a>. MIT Technology Review. Retrieved May 4, 2023.</cite></li></ul>

</li>

<li id="cite_note-quanta-hide-seek2-152"><a href="#cite_ref-quanta-hide-seek2_152-0">^</a> <cite id="CITEREFOrnes2019" class="citation web cs1">Ornes, Stephen (November 18, 2019). <a rel="nofollow" class="external text" href="https://www.quantamagazine.org/artificial-intelligence-discovers-tool-use-in-hide-and-seek-games-20191118/">"Playing Hide-and-Seek, Machines Invent New Tools"</a>. Quanta Magazine. <a rel="nofollow" class="external text" href="https://web.archive.org/web/20230210114137/https://www.quantamagazine.org/artificial-intelligence-discovers-tool-use-in-hide-and-seek-games-20191118/">Archived</a> from the original on February 10, 2023. Retrieved August 26, 2022.</cite>

</li>

<li id="cite_note-153"><a href="#cite_ref-153">^</a> <cite id="CITEREFBakerKanitscheiderMarkovWu2019" class="citation web cs1">Baker, Bowen; Kanitscheider, Ingmar; Markov, Todor; Wu, Yi; Powell, Glenn; McGrew, Bob; Mordatch, Igor (September 17, 2019). <a rel="nofollow" class="external text" href="https://openai.com/blog/emergent-tool-use/">"Emergent Tool Use from Multi-Agent Interaction"</a>. OpenAI. <a rel="nofollow" class="external text" href="https://web.archive.org/web/20220925043450/https://openai.com/blog/emergent-tool-use/">Archived</a> from the original on September 25, 2022. Retrieved August 26, 2022.</cite>

</li>

<li id="cite_note-154"><a href="#cite_ref-154">^</a> <cite id="CITEREFLuLuLangeFoerster2024" class="citation arxiv cs1">Lu, Chris; Lu, Cong; Lange, Robert Tjarko; Foerster, Jakob; Clune, Jeff; Ha, David (August 15, 2024). "The AI Scientist: Towards Fully Automated Open-Ended Scientific Discovery". <a href="ArXiv_(identifier)" class="mw-redirect" title="ArXiv (identifier)">arXiv</a>:<a rel="nofollow" class="external text" href="https://arxiv.org/abs/2408.06292">2408.06292</a> [<a rel="nofollow" class="external text" href="https://arxiv.org/archive/cs.AI">cs.AI</a>]. <q>In some cases, when The AI Scientist's experiments exceeded our imposed time limits, it attempted to edit the code to extend the time limit arbitrarily</q></cite>

</li>

<li id="cite_note-155"><a href="#cite_ref-155">^</a> <cite id="CITEREFEdwards2024" class="citation web cs1">Edwards, Benj (August 14, 2024). <a rel="nofollow" class="external text" href="https://arstechnica.com/information-technology/2024/08/research-ai-model-unexpectedly-modified-its-own-code-to-extend-runtime/">"Research AI model unexpectedly modified its own code to extend runtime"</a>. Ars Technica. Retrieved August 19, 2024.</cite>

</li>

<li id="cite_note-156"><a href="#cite_ref-156">^</a> <cite id="CITEREFShermer2017" class="citation web cs1">Shermer, Michael (March 1, 2017). <a rel="nofollow" class="external text" href="https://www.scientificamerican.com/article/artificial-intelligence-is-not-a-threat-mdash-yet/">"Artificial Intelligence Is Not a Threat—Yet"</a>. Scientific American. <a rel="nofollow" class="external text" href="https://web.archive.org/web/20171201051401/https://www.scientificamerican.com/article/artificial-intelligence-is-not-a-threat-mdash-yet/">Archived</a> from the original on December 1, 2017. Retrieved August 26, 2022.</cite>

</li>

<li id="cite_note-158"><a href="#cite_ref-158">^</a> <cite id="CITEREFBrownMannRyderSubbiah2020" class="citation arxiv cs1">Brown, Tom B.; Mann, Benjamin; Ryder, Nick; Subbiah, Melanie; Kaplan, Jared; Dhariwal, Prafulla; Neelakantan, Arvind; Shyam, Pranav; Sastry, Girish; Askell, Amanda; Agarwal, Sandhini; Herbert-Voss, Ariel; Krueger, Gretchen; Henighan, Tom; Child, Rewon (July 22, 2020). "Language Models are Few-Shot Learners". <a href="ArXiv_(identifier)" class="mw-redirect" title="ArXiv (identifier)">arXiv</a>:<a rel="nofollow" class="external text" href="https://arxiv.org/abs/2005.14165">2005.14165</a> [<a rel="nofollow" class="external text" href="https://arxiv.org/archive/cs.CL">cs.CL</a>].</cite>

<ul><li><cite id="CITEREFLaskinWangOhParisotto2022" class="citation arxiv cs1">Laskin, Michael; Wang, Luyu; Oh, Junhyuk; Parisotto, Emilio; Spencer, Stephen; Steigerwald, Richie; Strouse, D. J.; Hansen, Steven; Filos, Angelos; Brooks, Ethan; Gazeau, Maxime; Sahni, Himanshu; Singh, Satinder; Mnih, Volodymyr (October 25, 2022). "In-context Reinforcement Learning with Algorithm Distillation". <a href="ArXiv_(identifier)" class="mw-redirect" title="ArXiv (identifier)">arXiv</a>:<a rel="nofollow" class="external text" href="https://arxiv.org/abs/2210.14215">2210.14215</a> [<a rel="nofollow" class="external text" href="https://arxiv.org/archive/cs.LG">cs.LG</a>].</cite></li></ul>

</li>

<li id="cite_note-159"><a href="#cite_ref-159">^</a> <cite id="CITEREFMeloMáximoSomaCastro2025" class="citation journal cs1">Melo, Gabriel A.; Máximo, Marcos R. O. A.; Soma, Nei Y.; Castro, Paulo A. L. (2025). <a rel="nofollow" class="external text" href="https://www.ncbi.nlm.nih.gov/pmc/articles/PMC12050267">"Machines that halt resolve the undecidability of artificial intelligence alignment"</a>. Scientific Reports. 15 (1): 15591. <a href="Bibcode_(identifier)" class="mw-redirect" title="Bibcode (identifier)">Bibcode</a>:<a rel="nofollow" class="external text" href="https://ui.adsabs.harvard.edu/abs/2025NatSR..1515591M">2025NatSR..1515591M</a>. <a href="Doi_(identifier)" class="mw-redirect" title="Doi (identifier)">doi</a>:<a rel="nofollow" class="external text" href="https://doi.org/10.1038%2Fs41598-025-99060-2">10.1038/s41598-025-99060-2</a>. <a href="PMC_(identifier)" class="mw-redirect" title="PMC (identifier)">PMC</a> <a rel="nofollow" class="external text" href="https://www.ncbi.nlm.nih.gov/pmc/articles/PMC12050267">12050267</a>. <a href="PMID_(identifier)" class="mw-redirect" title="PMID (identifier)">PMID</a> <a rel="nofollow" class="external text" href="https://pubmed.ncbi.nlm.nih.gov/40320467">40320467</a>.</cite>

</li>

<li id="cite_note-GoalMisgeneralization-160">^ <a href="#cite_ref-GoalMisgeneralization_160-0">a</a> <a href="#cite_ref-GoalMisgeneralization_160-1">b</a> <a href="#cite_ref-GoalMisgeneralization_160-2">c</a> <cite id="CITEREFShahVarmaKumarPhuong2022" class="citation journal cs1">Shah, Rohin; Varma, Vikrant; Kumar, Ramana; Phuong, Mary; Krakovna, Victoria; Uesato, Jonathan; Kenton, Zac (November 2, 2022). <a rel="nofollow" class="external text" href="https://deepmindsafetyresearch.medium.com/goal-misgeneralisation-why-correct-specifications-arent-enough-for-correct-goals-cf96ebc60924">"Goal Misgeneralization: Why Correct Specifications Aren't Enough For Correct Goals"</a>. Medium. <a href="ArXiv_(identifier)" class="mw-redirect" title="ArXiv (identifier)">arXiv</a>:<a rel="nofollow" class="external text" href="https://arxiv.org/abs/2210.01790">2210.01790</a>. Retrieved April 2, 2023.</cite>

</li>

<li id="cite_note-rloamls-161">^ <a href="#cite_ref-rloamls_161-0">a</a> <a href="#cite_ref-rloamls_161-1">b</a> <cite id="CITEREFHubingervan_MerwijkMikulikSkalse2021" class="citation arxiv cs1">Hubinger, Evan; van Merwijk, Chris; Mikulik, Vladimir; Skalse, Joar; Garrabrant, Scott (December 1, 2021). "Risks from Learned Optimization in Advanced Machine Learning Systems". <a href="ArXiv_(identifier)" class="mw-redirect" title="ArXiv (identifier)">arXiv</a>:<a rel="nofollow" class="external text" href="https://arxiv.org/abs/1906.01820">1906.01820</a> [<a rel="nofollow" class="external text" href="https://arxiv.org/archive/cs.AI">cs.AI</a>].</cite>

</li>

<li id="cite_note-162"><a href="#cite_ref-162">^</a> <cite id="CITEREFZhangChanYanBose2022" class="citation journal cs1">Zhang, Xiaoge; Chan, Felix T.S.; Yan, Chao; Bose, Indranil (2022). <a rel="nofollow" class="external text" href="https://linkinghub.elsevier.com/retrieve/pii/S0167923622000719">"Towards risk-aware artificial intelligence and machine learning systems: An overview"</a>. Decision Support Systems. 159 113800. <a href="Doi_(identifier)" class="mw-redirect" title="Doi (identifier)">doi</a>:<a rel="nofollow" class="external text" href="https://doi.org/10.1016%2Fj.dss.2022.113800">10.1016/j.dss.2022.113800</a>. <a href="S2CID_(identifier)" class="mw-redirect" title="S2CID (identifier)">S2CID</a> <a rel="nofollow" class="external text" href="https://api.semanticscholar.org/CorpusID:248585546">248585546</a>.</cite>

</li>

<li id="cite_note-163"><a href="#cite_ref-163">^</a> <cite id="CITEREFDemskiGarrabrant2020" class="citation arxiv cs1">Demski, Abram; Garrabrant, Scott (October 6, 2020). "Embedded Agency". <a href="ArXiv_(identifier)" class="mw-redirect" title="ArXiv (identifier)">arXiv</a>:<a rel="nofollow" class="external text" href="https://arxiv.org/abs/1902.09469">1902.09469</a> [<a rel="nofollow" class="external text" href="https://arxiv.org/archive/cs.AI">cs.AI</a>].</cite>

</li>

<li id="cite_note-causal_influence2-164">^ <a href="#cite_ref-causal_influence2_164-0">a</a> <a href="#cite_ref-causal_influence2_164-1">b</a> <cite id="CITEREFEverittOrtegaBarnesLegg2019" class="citation arxiv cs1">Everitt, Tom; Ortega, Pedro A.; Barnes, Elizabeth; Legg, Shane (September 6, 2019). "Understanding Agent Incentives using Causal Influence Diagrams. Part I: Single Action Settings". <a href="ArXiv_(identifier)" class="mw-redirect" title="ArXiv (identifier)">arXiv</a>:<a rel="nofollow" class="external text" href="https://arxiv.org/abs/1902.09980">1902.09980</a> [<a rel="nofollow" class="external text" href="https://arxiv.org/archive/cs.AI">cs.AI</a>].</cite>

</li>

<li id="cite_note-:323-165">^ <a href="#cite_ref-:323_165-0">a</a> <a href="#cite_ref-:323_165-1">b</a> <cite id="CITEREFCohenHutterOsborne2022" class="citation journal cs1">Cohen, Michael K.; Hutter, Marcus; Osborne, Michael A. (August 29, 2022). <a rel="nofollow" class="external text" href="https://onlinelibrary.wiley.com/doi/10.1002/aaai.12064">"Advanced artificial agents intervene in the provision of reward"</a>. AI Magazine. 43 (3): 282–293. <a href="Doi_(identifier)" class="mw-redirect" title="Doi (identifier)">doi</a>:<a rel="nofollow" class="external text" href="https://doi.org/10.1002%2Faaai.12064">10.1002/aaai.12064</a>. <a href="ISSN_(identifier)" class="mw-redirect" title="ISSN (identifier)">ISSN</a> <a rel="nofollow" class="external text" href="https://search.worldcat.org/issn/0738-4602">0738-4602</a>. <a href="S2CID_(identifier)" class="mw-redirect" title="S2CID (identifier)">S2CID</a> <a rel="nofollow" class="external text" href="https://api.semanticscholar.org/CorpusID:235489158">235489158</a>. <a rel="nofollow" class="external text" href="https://web.archive.org/web/20230210153534/https://onlinelibrary.wiley.com/doi/10.1002/aaai.12064">Archived</a> from the original on February 10, 2023. Retrieved September 6, 2022.</cite>

</li>

<li id="cite_note-Hadfield-Menell2019-166"><a href="#cite_ref-Hadfield-Menell2019_166-0">^</a> <cite id="CITEREFHadfield-MenellHadfield2019" class="citation conference cs1">Hadfield-Menell, Dylan; Hadfield, Gillian K (2019). "Incomplete contracting and AI alignment". Proceedings of the 2019 AAAI/ACM Conference on AI, Ethics, and Society. pp. 417–422.</cite>

</li>

<li id="cite_note-Hanson2019-167"><a href="#cite_ref-Hanson2019_167-0">^</a> <cite id="CITEREFHanson2019" class="citation web cs1">Hanson, Robin (April 10, 2019). <a rel="nofollow" class="external text" href="https://www.overcomingbias.com/p/agency-failure-ai-apocalypsehtml">"Agency Failure or AI Apocalypse?"</a>. Overcoming Bias. Retrieved September 20, 2023.</cite>

</li>

<li id="cite_note-168"><a href="#cite_ref-168">^</a> <cite id="CITEREFHamilton2020" class="citation cs2">Hamilton, Andy (2020), <a rel="nofollow" class="external text" href="https://plato.stanford.edu/entries/conservatism/">"Conservatism"</a>, in Zalta, Edward N. (ed.), The Stanford Encyclopedia of Philosophy (Spring 2020 ed.), Metaphysics Research Lab, Stanford University, retrieved October 16, 2024</cite>

</li>

<li id="cite_note-169"><a href="#cite_ref-169">^</a> <cite id="CITEREFTaylorYudkowskyLaVictoireCritch2016" class="citation web cs1">Taylor, Jessica; Yudkowsky, Eliezer; LaVictoire, Patrick; Critch, Andrew (July 27, 2016). <a rel="nofollow" class="external text" href="https://intelligence.org/files/AlignmentMachineLearning.pdf">"Alignment for Advanced Machine Learning Systems"</a> (PDF).</cite>

</li>

<li id="cite_note-170"><a href="#cite_ref-170">^</a> <cite id="CITEREFBengio2024" class="citation web cs1">Bengio, Yoshua (February 26, 2024). <a rel="nofollow" class="external text" href="https://yoshuabengio.org/2024/02/26/towards-a-cautious-scientist-ai-with-convergent-safety-bounds/">"Towards a Cautious Scientist AI with Convergent Safety Bounds"</a>.</cite>

</li>

<li id="cite_note-171"><a href="#cite_ref-171">^</a> <cite id="CITEREFCohenHutter2020" class="citation journal cs1">Cohen, Michael; Hutter, Marcus (2020). <a rel="nofollow" class="external text" href="https://proceedings.mlr.press/v125/cohen20a/cohen20a.pdf">"Pessimism about unknown unknowns inspires conservatism"</a> (PDF). Proceedings of Machine Learning Research. 125: 1344–1373. <a href="ArXiv_(identifier)" class="mw-redirect" title="ArXiv (identifier)">arXiv</a>:<a rel="nofollow" class="external text" href="https://arxiv.org/abs/2006.08753">2006.08753</a>.</cite>

</li>

<li id="cite_note-172"><a href="#cite_ref-172">^</a> <cite id="CITEREFLiuReyzinZiebart2015" class="citation journal cs1">Liu, Anqi; Reyzin, Lev; Ziebart, Brian (February 21, 2015). <a rel="nofollow" class="external text" href="https://ojs.aaai.org/index.php/AAAI/article/view/9609">"Shift-Pessimistic Active Learning Using Robust Bias-Aware Prediction"</a>. Proceedings of the AAAI Conference on Artificial Intelligence. 29 (1). <a href="Doi_(identifier)" class="mw-redirect" title="Doi (identifier)">doi</a>:<a rel="nofollow" class="external text" href="https://doi.org/10.1609%2Faaai.v29i1.9609">10.1609/aaai.v29i1.9609</a>. <a href="ISSN_(identifier)" class="mw-redirect" title="ISSN (identifier)">ISSN</a> <a rel="nofollow" class="external text" href="https://search.worldcat.org/issn/2374-3468">2374-3468</a>.</cite>

</li>

<li id="cite_note-173"><a href="#cite_ref-173">^</a> <cite id="CITEREFLiuShenCuiZhou2021" class="citation journal cs1">Liu, Jiashuo; Shen, Zheyan; Cui, Peng; Zhou, Linjun; Kuang, Kun; Li, Bo; Lin, Yishi (May 18, 2021). <a rel="nofollow" class="external text" href="https://ojs.aaai.org/index.php/AAAI/article/view/17050">"Stable Adversarial Learning under Distributional Shifts"</a>. Proceedings of the AAAI Conference on Artificial Intelligence. 35 (10): 8662–8670. <a href="ArXiv_(identifier)" class="mw-redirect" title="ArXiv (identifier)">arXiv</a>:<a rel="nofollow" class="external text" href="https://arxiv.org/abs/2006.04414">2006.04414</a>. <a href="Doi_(identifier)" class="mw-redirect" title="Doi (identifier)">doi</a>:<a rel="nofollow" class="external text" href="https://doi.org/10.1609%2Faaai.v35i10.17050">10.1609/aaai.v35i10.17050</a>. <a href="ISSN_(identifier)" class="mw-redirect" title="ISSN (identifier)">ISSN</a> <a rel="nofollow" class="external text" href="https://search.worldcat.org/issn/2374-3468">2374-3468</a>.</cite>

</li>

<li id="cite_note-174"><a href="#cite_ref-174">^</a> <cite id="CITEREFRoyXuPokutta2017" class="citation journal cs1">Roy, Aurko; Xu, Huan; Pokutta, Sebastian (2017). <a rel="nofollow" class="external text" href="https://papers.nips.cc/paper_files/paper/2017/hash/84c6494d30851c63a55cdb8cb047fadd-Abstract.html">"Reinforcement Learning under Model Mismatch"</a>. Advances in Neural Information Processing Systems. 30. Curran Associates, Inc.</cite>

</li>

<li id="cite_note-175"><a href="#cite_ref-175">^</a> <cite id="CITEREFPintoDavidsonSukthankarGupta2017" class="citation journal cs1">Pinto, Lerrel; Davidson, James; Sukthankar, Rahul; Gupta, Abhinav (July 17, 2017). <a rel="nofollow" class="external text" href="https://proceedings.mlr.press/v70/pinto17a.html">"Robust Adversarial Reinforcement Learning"</a>. Proceedings of the 34th International Conference on Machine Learning. PMLR: 2817–2826.</cite>

</li>

<li id="cite_note-176"><a href="#cite_ref-176">^</a> <cite id="CITEREFWangZou2021" class="citation journal cs1">Wang, Yue; Zou, Shaofeng (2021). <a rel="nofollow" class="external text" href="https://proceedings.neurips.cc/paper/2021/hash/3a4496776767aaa99f9804d0905fe584-Abstract.html">"Online Robust Reinforcement Learning with Model Uncertainty"</a>. Advances in Neural Information Processing Systems. 34. Curran Associates, Inc.: 7193–7206. <a href="ArXiv_(identifier)" class="mw-redirect" title="ArXiv (identifier)">arXiv</a>:<a rel="nofollow" class="external text" href="https://arxiv.org/abs/2109.14523">2109.14523</a>.</cite>

</li>

<li id="cite_note-177"><a href="#cite_ref-177">^</a> <cite id="CITEREFBlanchetLuZhangZhong2023" class="citation journal cs1">Blanchet, Jose; Lu, Miao; Zhang, Tong; Zhong, Han (December 15, 2023). <a rel="nofollow" class="external text" href="https://proceedings.neurips.cc/paper_files/paper/2023/hash/d31b005d817e9c635ec8ffb0fb90190e-Abstract-Conference.html">"Double Pessimism is Provably Efficient for Distributionally Robust Offline Reinforcement Learning: Generic Algorithm and Robust Partial Coverage"</a>. Advances in Neural Information Processing Systems. 36: 66845–66859. <a href="ArXiv_(identifier)" class="mw-redirect" title="ArXiv (identifier)">arXiv</a>:<a rel="nofollow" class="external text" href="https://arxiv.org/abs/2305.09659">2305.09659</a>.</cite>

</li>

<li id="cite_note-178"><a href="#cite_ref-178">^</a> <cite id="CITEREFLevineKumarTuckerFu2020" class="citation arxiv cs1">Levine, Sergey; Kumar, Aviral; Tucker, George; Fu, Justin (November 1, 2020). "Offline Reinforcement Learning: Tutorial, Review, and Perspectives on Open Problems". <a href="ArXiv_(identifier)" class="mw-redirect" title="ArXiv (identifier)">arXiv</a>:<a rel="nofollow" class="external text" href="https://arxiv.org/abs/2005.01643">2005.01643</a> [<a rel="nofollow" class="external text" href="https://arxiv.org/archive/cs.LG">cs.LG</a>].</cite>

</li>

<li id="cite_note-179"><a href="#cite_ref-179">^</a> <cite id="CITEREFRigterLacerdaHawes2022" class="citation journal cs1">Rigter, Marc; Lacerda, Bruno; Hawes, Nick (December 6, 2022). <a rel="nofollow" class="external text" href="https://proceedings.neurips.cc/paper_files/paper/2022/hash/6691c5e4a199b72dffd9c90acb63bcd6-Abstract-Conference.html">"RAMBO-RL: Robust Adversarial Model-Based Offline Reinforcement Learning"</a>. Advances in Neural Information Processing Systems. 35: 16082–16097. <a href="ArXiv_(identifier)" class="mw-redirect" title="ArXiv (identifier)">arXiv</a>:<a rel="nofollow" class="external text" href="https://arxiv.org/abs/2204.12581">2204.12581</a>.</cite>

</li>

<li id="cite_note-180"><a href="#cite_ref-180">^</a> <cite id="CITEREFGuoYunfengGeng2022" class="citation journal cs1">Guo, Kaiyang; Yunfeng, Shao; Geng, Yanhui (December 6, 2022). <a rel="nofollow" class="external text" href="https://proceedings.neurips.cc/paper_files/paper/2022/hash/03469b1a66e351b18272be23baf3b809-Abstract-Conference.html">"Model-Based Offline Reinforcement Learning with Pessimism-Modulated Dynamics Belief"</a>. Advances in Neural Information Processing Systems. 35: 449–461. <a href="ArXiv_(identifier)" class="mw-redirect" title="ArXiv (identifier)">arXiv</a>:<a rel="nofollow" class="external text" href="https://arxiv.org/abs/2210.06692">2210.06692</a>.</cite>

</li>

<li id="cite_note-181"><a href="#cite_ref-181">^</a> <cite id="CITEREFCosteAnwarKirkKrueger2024" class="citation journal cs1">Coste, Thomas; Anwar, Usman; Kirk, Robert; Krueger, David (January 16, 2024). <a rel="nofollow" class="external text" href="https://openreview.net/forum?id=dcjtMYkpXx">"Reward Model Ensembles Help Mitigate Overoptimization"</a>. International Conference on Learning Representations. <a href="ArXiv_(identifier)" class="mw-redirect" title="ArXiv (identifier)">arXiv</a>:<a rel="nofollow" class="external text" href="https://arxiv.org/abs/2310.02743">2310.02743</a>.</cite>

</li>

<li id="cite_note-182"><a href="#cite_ref-182">^</a> <cite id="CITEREFLiuLuZhangLiu2024" class="citation arxiv cs1">Liu, Zhihan; Lu, Miao; Zhang, Shenao; Liu, Boyi; Guo, Hongyi; Yang, Yingxiang; Blanchet, Jose; Wang, Zhaoran (May 26, 2024). "Provably Mitigating Overoptimization in RLHF: Your SFT Loss is Implicitly an Adversarial Regularizer". <a href="ArXiv_(identifier)" class="mw-redirect" title="ArXiv (identifier)">arXiv</a>:<a rel="nofollow" class="external text" href="https://arxiv.org/abs/2405.16436">2405.16436</a> [<a rel="nofollow" class="external text" href="https://arxiv.org/archive/cs.LG">cs.LG</a>].</cite>

</li>

<li id="cite_note-183"><a href="#cite_ref-183">^</a> <cite id="CITEREFCohenHutterNanda2022" class="citation journal cs1">Cohen, Michael K.; Hutter, Marcus; Nanda, Neel (2022). <a rel="nofollow" class="external text" href="https://jmlr.org/papers/v23/21-0618.html">"Fully General Online Imitation Learning"</a>. Journal of Machine Learning Research. 23 (334): 1–30. <a href="ArXiv_(identifier)" class="mw-redirect" title="ArXiv (identifier)">arXiv</a>:<a rel="nofollow" class="external text" href="https://arxiv.org/abs/2102.08686">2102.08686</a>. <a href="ISSN_(identifier)" class="mw-redirect" title="ISSN (identifier)">ISSN</a> <a rel="nofollow" class="external text" href="https://search.worldcat.org/issn/1533-7928">1533-7928</a>.</cite>

</li>

<li id="cite_note-184"><a href="#cite_ref-184">^</a> <cite id="CITEREFChangUeharaSreenivasKidambi2021" class="citation journal cs1">Chang, Jonathan; Uehara, Masatoshi; Sreenivas, Dhruv; Kidambi, Rahul; Sun, Wen (2021). <a rel="nofollow" class="external text" href="https://proceedings.neurips.cc/paper_files/paper/2021/hash/07d5938693cc3903b261e1a3844590ed-Abstract.html">"Mitigating Covariate Shift in Imitation Learning via Offline Data With Partial Coverage"</a>. Advances in Neural Information Processing Systems. 34. Curran Associates, Inc.: 965–979.</cite>

</li>

<li id="cite_note-185"><a href="#cite_ref-185">^</a> <cite id="CITEREFBoydVandenberghe2023" class="citation book cs1">Boyd, Stephen P.; Vandenberghe, Lieven (2023). Convex optimization (Version 29 ed.). Cambridge New York Melbourne New Delhi Singapore: Cambridge University Press. <a href="ISBN_(identifier)" class="mw-redirect" title="ISBN (identifier)">ISBN</a> <bdi>978-0-521-83378-3</bdi>.</cite>

</li>

<li id="cite_note-186"><a href="#cite_ref-186">^</a> <cite id="CITEREFKosoyAppel2021" class="citation journal cs1">Kosoy, Vanessa; Appel, Alexander (November 30, 2021). <a rel="nofollow" class="external text" href="https://www.alignmentforum.org/posts/gHgs2e2J5azvGFatb/infra-bayesian-physicalism-a-formal-theory-of-naturalized">"Infra-Bayesian physicalism: a formal theory of naturalized induction"</a>. Alignment Forum.</cite>

</li>

<li id="cite_note-187"><a href="#cite_ref-187">^</a> <cite class="citation web cs1"><a rel="nofollow" class="external text" href="https://web.archive.org/web/20230216065407/https://www.un.org/en/content/common-agenda-report/">"UN Secretary-General's report on "Our Common Agenda""</a>. 2021. p. 63. Archived from <a rel="nofollow" class="external text" href="https://www.un.org/en/content/common-agenda-report/">the original</a> on February 16, 2023. <q>[T]he Compact could also promote regulation of artificial intelligence to ensure that this is aligned with shared global values</q></cite>

</li>

<li id="cite_note-188"><a href="#cite_ref-188">^</a> <cite id="CITEREFThe_National_New_Generation_Artificial_Intelligence_Governance_Specialist_Committee2021" class="citation web cs1">The National New Generation Artificial Intelligence Governance Specialist Committee (October 12, 2021) [2021-09-25]. <a rel="nofollow" class="external text" href="https://cset.georgetown.edu/publication/ethical-norms-for-new-generation-artificial-intelligence-released/">"Ethical Norms for New Generation Artificial Intelligence Released"</a>. Translated by <a href="Center_for_Security_and_Emerging_Technology" title="Center for Security and Emerging Technology">Center for Security and Emerging Technology</a>. <a rel="nofollow" class="external text" href="https://web.archive.org/web/20230210114220/https://cset.georgetown.edu/publication/ethical-norms-for-new-generation-artificial-intelligence-released/">Archived</a> from the original on February 10, 2023.</cite>

</li>

<li id="cite_note-189"><a href="#cite_ref-189">^</a> <cite id="CITEREFRichardson2021" class="citation news cs1">Richardson, Tim (September 22, 2021). <a rel="nofollow" class="external text" href="https://www.theregister.com/2021/09/22/uk_10_year_national_ai_strategy/">"UK publishes National Artificial Intelligence Strategy"</a>. The Register. <a rel="nofollow" class="external text" href="https://web.archive.org/web/20230210114137/https://www.theregister.com/2021/09/22/uk_10_year_national_ai_strategy/">Archived</a> from the original on February 10, 2023. Retrieved November 14, 2021.</cite>

</li>

<li id="cite_note-190"><a href="#cite_ref-190">^</a> <cite class="citation web cs1"><a rel="nofollow" class="external text" href="https://web.archive.org/web/20230210114139/https://www.gov.uk/government/publications/national-ai-strategy/national-ai-strategy-html-version">"The National AI Strategy of the UK"</a>. 2021. Archived from <a rel="nofollow" class="external text" href="https://www.gov.uk/government/publications/national-ai-strategy/national-ai-strategy-html-version">the original</a> on February 10, 2023. <q>The government takes the long term risk of non-aligned Artificial General Intelligence, and the unforeseeable changes that it would mean for the UK and the world, seriously.</q></cite>

</li>

<li id="cite_note-191"><a href="#cite_ref-191">^</a> <cite class="citation web cs1"><a rel="nofollow" class="external text" href="https://web.archive.org/web/20230210114139/https://www.gov.uk/government/publications/national-ai-strategy/national-ai-strategy-html-version">"The National AI Strategy of the UK"</a>. 2021. actions 9 and 10 of the section "Pillar 3 – Governing AI Effectively". Archived from <a rel="nofollow" class="external text" href="https://www.gov.uk/government/publications/national-ai-strategy/national-ai-strategy-html-version">the original</a> on February 10, 2023.</cite>

</li>

<li id="cite_note-192"><a href="#cite_ref-192">^</a> <cite class="citation book cs1"><a rel="nofollow" class="external text" href="https://www.nscai.gov/wp-content/uploads/2021/03/Full-Report-Digital-1.pdf">NSCAI Final Report</a> (PDF). Washington, DC: The National Security Commission on Artificial Intelligence. 2021. <a rel="nofollow" class="external text" href="https://web.archive.org/web/20230215110858/https://www.nscai.gov/wp-content/uploads/2021/03/Full-Report-Digital-1.pdf">Archived</a> (PDF) from the original on February 15, 2023. Retrieved October 17, 2022.</cite>

</li>

<li id="cite_note-193"><a href="#cite_ref-193">^</a> <cite id="CITEREFRobert_Lee_Poe2023" class="citation arxiv cs1">Robert Lee Poe (2023). "Why Fair Automated Hiring Systems Breach EU Non-Discrimination Law". <a href="ArXiv_(identifier)" class="mw-redirect" title="ArXiv (identifier)">arXiv</a>:<a rel="nofollow" class="external text" href="https://arxiv.org/abs/2311.03900">2311.03900</a> [<a rel="nofollow" class="external text" href="https://arxiv.org/archive/cs.CY">cs.CY</a>].</cite>

</li>

<li id="cite_note-194"><a href="#cite_ref-194">^</a> <cite id="CITEREFDe_Vos2020" class="citation journal cs1">De Vos, Marc (2020). <a rel="nofollow" class="external text" href="https://doi.org/10.1177/1358229120927947">"The European Court of Justice and the march towards substantive equality in European Union anti-discrimination law"</a>. International Journal of Discrimination and the Law. 20: 62–87. <a href="Doi_(identifier)" class="mw-redirect" title="Doi (identifier)">doi</a>:<a rel="nofollow" class="external text" href="https://doi.org/10.1177%2F1358229120927947">10.1177/1358229120927947</a>.</cite>

</li>

<li id="cite_note-195"><a href="#cite_ref-195">^</a> <cite id="CITEREFIrvingAskell2016" class="citation journal cs1">Irving, Geoffrey; Askell, Amanda (June 9, 2016). "Chern number in Ising models with spatially modulated real and complex fields". Physical Review A. 94 (5): 052113. <a href="ArXiv_(identifier)" class="mw-redirect" title="ArXiv (identifier)">arXiv</a>:<a rel="nofollow" class="external text" href="https://arxiv.org/abs/1606.03535">1606.03535</a>. <a href="Bibcode_(identifier)" class="mw-redirect" title="Bibcode (identifier)">Bibcode</a>:<a rel="nofollow" class="external text" href="https://ui.adsabs.harvard.edu/abs/2016PhRvA..94e2113L">2016PhRvA..94e2113L</a>. <a href="Doi_(identifier)" class="mw-redirect" title="Doi (identifier)">doi</a>:<a rel="nofollow" class="external text" href="https://doi.org/10.1103%2FPhysRevA.94.052113">10.1103/PhysRevA.94.052113</a>. <a href="S2CID_(identifier)" class="mw-redirect" title="S2CID (identifier)">S2CID</a> <a rel="nofollow" class="external text" href="https://api.semanticscholar.org/CorpusID:118699363">118699363</a>.</cite>

</li>

<li id="cite_note-196"><a href="#cite_ref-196">^</a> <cite id="CITEREFMitelutSmithVamplew2023" class="citation arxiv cs1">Mitelut, Catalin; Smith, Ben; Vamplew, Peter (May 30, 2023). "Intent-aligned AI systems deplete human agency: the need for agency foundations research in AI safety". <a href="ArXiv_(identifier)" class="mw-redirect" title="ArXiv (identifier)">arXiv</a>:<a rel="nofollow" class="external text" href="https://arxiv.org/abs/2305.19223">2305.19223</a> [<a rel="nofollow" class="external text" href="https://arxiv.org/archive/cs.AI">cs.AI</a>].</cite>

</li>

<li id="cite_note-197"><a href="#cite_ref-197">^</a> <cite id="CITEREFGabriel2020" class="citation journal cs1">Gabriel, Iason (September 1, 2020). <a rel="nofollow" class="external text" href="https://doi.org/10.1007%2Fs11023-020-09539-2">"Artificial Intelligence, Values, and Alignment"</a>. Minds and Machines. 30 (3): 411–437. <a href="ArXiv_(identifier)" class="mw-redirect" title="ArXiv (identifier)">arXiv</a>:<a rel="nofollow" class="external text" href="https://arxiv.org/abs/2001.09768">2001.09768</a>. <a href="Doi_(identifier)" class="mw-redirect" title="Doi (identifier)">doi</a>:<a rel="nofollow" class="external text" href="https://doi.org/10.1007%2Fs11023-020-09539-2">10.1007/s11023-020-09539-2</a>. <a href="S2CID_(identifier)" class="mw-redirect" title="S2CID (identifier)">S2CID</a> <a rel="nofollow" class="external text" href="https://api.semanticscholar.org/CorpusID:210920551">210920551</a>.</cite>

</li>

<li id="cite_note-198"><a href="#cite_ref-198">^</a> <cite id="CITEREFRussell2019" class="citation book cs1">Russell, Stuart J. (2019). <a rel="nofollow" class="external text" href="https://www.penguinrandomhouse.com/books/566677/human-compatible-by-stuart-russell/">Human Compatible: Artificial Intelligence and the Problem of Control</a>. Penguin Random House.</cite>

</li>

<li id="cite_note-199"><a href="#cite_ref-199">^</a> <cite id="CITEREFDafoe2019" class="citation journal cs1">Dafoe, Allan (2019). <a rel="nofollow" class="external text" href="https://www.nature.com/articles/s41586-019-1420-6">"AI policy: A roadmap"</a>. Nature.</cite>

</li>

</ol></div></div>

<div class="mw-heading mw-heading2"><h2 id="Further_reading">Further reading</h2></div>

<ul><li><cite id="CITEREFBrockman2019" class="citation book cs1"><a href="John_Brockman_(literary_agent)" title="John Brockman (literary agent)">Brockman, John</a>, ed. (2019). <a href="Possible_Minds" title="Possible Minds">Possible Minds: Twenty-five Ways of Looking at AI</a> (Kindle ed.). Penguin Press. <a href="ISBN_(identifier)" class="mw-redirect" title="ISBN (identifier)">ISBN</a> <bdi>978-0525557999</bdi>.</cite><code class="cs1-code">{{cite book}}</code>: CS1 maint: ref duplicates default (link)</li>

<li><cite id="CITEREFNgoChanMindermann2023" class="citation arxiv cs1">Ngo, Richard; et al. (2023). "The Alignment Problem from a Deep Learning Perspective". <a href="ArXiv_(identifier)" class="mw-redirect" title="ArXiv (identifier)">arXiv</a>:<a rel="nofollow" class="external text" href="https://arxiv.org/abs/2209.00626">2209.00626</a> [<a rel="nofollow" class="external text" href="https://arxiv.org/archive/cs.AI">cs.AI</a>].</cite></li>

<li><cite id="CITEREFJiQiuChen2023" class="citation arxiv cs1">Ji, Jiaming; et al. (2023). "AI Alignment: A Comprehensive Survey". <a href="ArXiv_(identifier)" class="mw-redirect" title="ArXiv (identifier)">arXiv</a>:<a rel="nofollow" class="external text" href="https://arxiv.org/abs/2310.19852">2310.19852</a> [<a rel="nofollow" class="external text" href="https://arxiv.org/archive/cs.AI">cs.AI</a>].</cite></li></ul>

<div class="mw-heading mw-heading2"><h2 id="External_links">External links</h2></div>

<ul><li><a rel="nofollow" class="external text" href="https://docs.google.com/spreadsheets/d/e/2PACX-1vRPiprOaC3HsCf5Tuum8bRfzYUiKLRqJmbOoC-32JorNdfyTiRRsR7Ea5eWtvsWzuxo8bjOxCG84dAg/pubhtml">Specification gaming examples in AI</a>, via <a rel="nofollow" class="external text" href="https://deepmind.com/blog/article/Specification-gaming-the-flip-side-of-AI-ingenuity">DeepMind</a></li></ul>

/* start https://en.wikipedia.org/ */

.mw-parser-output .navbox{box-sizing:border-box;border:1px solid #a2a9b1;width:100%;clear:both;font-size:88%;text-align:center;padding:1px;margin:1em auto 0}.mw-parser-output .navbox .navbox{margin-top:0}.mw-parser-output .navbox+.navbox,.mw-parser-output .navbox+.navbox-styles+.navbox{margin-top:-1px}.mw-parser-output .navbox-inner,.mw-parser-output .navbox-subgroup{width:100%}.mw-parser-output .navbox-group,.mw-parser-output .navbox-title,.mw-parser-output .navbox-abovebelow{padding:0.25em 1em;line-height:1.5em;text-align:center}.mw-parser-output .navbox-group{white-space:nowrap;text-align:right}.mw-parser-output .navbox,.mw-parser-output .navbox-subgroup{background-color:#fdfdfd}.mw-parser-output .navbox-list{line-height:1.5em;border-color:#fdfdfd}.mw-parser-output .navbox-list-with-group{text-align:left;border-left-width:2px;border-left-style:solid}.mw-parser-output tr+tr>.navbox-abovebelow,.mw-parser-output tr+tr>.navbox-group,.mw-parser-output tr+tr>.navbox-image,.mw-parser-output tr+tr>.navbox-list{border-top:2px solid #fdfdfd}.mw-parser-output .navbox-title{background-color:#ccf}.mw-parser-output .navbox-abovebelow,.mw-parser-output .navbox-group,.mw-parser-output .navbox-subgroup .navbox-title{background-color:#ddf}.mw-parser-output .navbox-subgroup .navbox-group,.mw-parser-output .navbox-subgroup .navbox-abovebelow{background-color:#e6e6ff}.mw-parser-output .navbox-even{background-color:#f7f7f7}.mw-parser-output .navbox-odd{background-color:transparent}.mw-parser-output .navbox .hlist td dl,.mw-parser-output .navbox .hlist td ol,.mw-parser-output .navbox .hlist td ul,.mw-parser-output .navbox td.hlist dl,.mw-parser-output .navbox td.hlist ol,.mw-parser-output .navbox td.hlist ul{padding:0.125em 0}.mw-parser-output .navbox .navbar{display:block;font-size:100%}.mw-parser-output .navbox-title .navbar{float:left;text-align:left;margin-right:0.5em}body.skin--responsive .mw-parser-output .navbox-image img{max-width:none!important}@media print{body.ns-0 .mw-parser-output .navbox{display:none!important}}

/* end https://en.wikipedia.org/ */

</style></div><div role="navigation" class="navbox" aria-labelledby="Existential_risk_from_artificial_intelligence277" style="padding:3px"><table class="nowraplinks mw-collapsible expanded navbox-inner" style="border-spacing:0;background:transparent;color:inherit"><tbody><tr><th scope="col" class="navbox-title" colspan="2"><div id="Existential_risk_from_artificial_intelligence277" style="font-size:114%;margin:0 4em"><a href="Existential_risk_from_artificial_intelligence" title="Existential risk from artificial intelligence">Existential risk</a> from <a href="Artificial_intelligence" title="Artificial intelligence">artificial intelligence</a></div></th></tr><tr><th scope="row" class="navbox-group" style="width:1%">Concepts</th><td class="navbox-list-with-group navbox-list navbox-odd hlist" style="width:100%;padding:0"><div style="padding:0 0.25em">

<li><a href="AI_capability_control" title="AI capability control">AI capability control</a></li>

<li><a href="AI_safety" title="AI safety">AI safety</a></li>

<li><a href="AI_takeover" title="AI takeover">AI takeover</a></li>

<li><a href="Consequentialism" title="Consequentialism">Consequentialism</a></li>

<li><a href="Effective_accelerationism" title="Effective accelerationism">Effective accelerationism</a></li>

<li><a href="Ethics_of_artificial_intelligence" title="Ethics of artificial intelligence">Ethics of artificial intelligence</a></li>

<li><a href="Existential_risk_from_artificial_intelligence" title="Existential risk from artificial intelligence">Existential risk from artificial intelligence</a></li>

<li><a href="Friendly_artificial_intelligence" title="Friendly artificial intelligence">Friendly artificial intelligence</a></li>

<li><a href="Instrumental_convergence" title="Instrumental convergence">Instrumental convergence</a></li>

<li><a href="Vulnerable_world_hypothesis" title="Vulnerable world hypothesis">Vulnerable world hypothesis</a></li>

<li><a href="Intelligence_explosion" class="mw-redirect" title="Intelligence explosion">Intelligence explosion</a></li>

<li><a href="Longtermism" title="Longtermism">Longtermism</a></li>

<li><a href="Machine_ethics" title="Machine ethics">Machine ethics</a></li>

<li><a href="Risk_of_astronomical_suffering" title="Risk of astronomical suffering">Suffering risks</a></li>

<li><a href="Superintelligence" title="Superintelligence">Superintelligence</a></li>

<li><a href="Technological_singularity" title="Technological singularity">Technological singularity</a></li></ul>

</div></td></tr><tr><th scope="row" class="navbox-group" style="width:1%">Organizations</th><td class="navbox-list-with-group navbox-list navbox-even hlist" style="width:100%;padding:0"><div style="padding:0 0.25em">

<ul><li><a href="Alignment_Research_Center" title="Alignment Research Center">Alignment Research Center</a></li>

<li><a href="Center_for_AI_Safety" title="Center for AI Safety">Center for AI Safety</a></li>

<li><a href="Center_for_Applied_Rationality" title="Center for Applied Rationality">Center for Applied Rationality</a></li>

<li><a href="Center_for_Human-Compatible_Artificial_Intelligence" title="Center for Human-Compatible Artificial Intelligence">Center for Human-Compatible Artificial Intelligence</a></li>

<li><a href="Centre_for_the_Study_of_Existential_Risk" title="Centre for the Study of Existential Risk">Centre for the Study of Existential Risk</a></li>

<li><a href="EleutherAI" title="EleutherAI">EleutherAI</a></li>

<li><a href="Future_of_Humanity_Institute" title="Future of Humanity Institute">Future of Humanity Institute</a></li>

<li><a href="Future_of_Life_Institute" title="Future of Life Institute">Future of Life Institute</a></li>

<li><a href="Google_DeepMind" title="Google DeepMind">Google DeepMind</a></li>

<li><a href="Humanity%2B" title="Humanity+">Humanity+</a></li>

<li><a href="Institute_for_Ethics_and_Emerging_Technologies" title="Institute for Ethics and Emerging Technologies">Institute for Ethics and Emerging Technologies</a></li>

<li><a href="Leverhulme_Centre_for_the_Future_of_Intelligence" title="Leverhulme Centre for the Future of Intelligence">Leverhulme Centre for the Future of Intelligence</a></li>

<li><a href="Machine_Intelligence_Research_Institute" title="Machine Intelligence Research Institute">Machine Intelligence Research Institute</a></li>

<li><a href="OpenAI" title="OpenAI">OpenAI</a></li>

<li><a href="Safe_Superintelligence_Inc." title="Safe Superintelligence Inc.">Safe Superintelligence</a></li></ul>

</div></td></tr><tr><th scope="row" class="navbox-group" style="width:1%">People</th><td class="navbox-list-with-group navbox-list navbox-odd hlist" style="width:100%;padding:0"><div style="padding:0 0.25em">

<ul><li><a href="Slate_Star_Codex" title="Slate Star Codex">Scott Alexander</a></li>

<li><a href="Sam_Altman" title="Sam Altman">Sam Altman</a></li>

<li><a href="Yoshua_Bengio" title="Yoshua Bengio">Yoshua Bengio</a></li>

<li><a href="Nick_Bostrom" title="Nick Bostrom">Nick Bostrom</a></li>

<li><a href="Paul_Christiano_(researcher)" class="mw-redirect" title="Paul Christiano (researcher)">Paul Christiano</a></li>

<li><a href="K._Eric_Drexler" title="K. Eric Drexler">Eric Drexler</a></li>

<li><a href="Sam_Harris" title="Sam Harris">Sam Harris</a></li>

<li><a href="Stephen_Hawking" title="Stephen Hawking">Stephen Hawking</a></li>

<li><a href="Dan_Hendrycks" title="Dan Hendrycks">Dan Hendrycks</a></li>

<li><a href="Geoffrey_Hinton" title="Geoffrey Hinton">Geoffrey Hinton</a></li>

<li><a href="Shane_Legg" title="Shane Legg">Shane Legg</a></li>

<li><a href="Steve_Omohundro" title="Steve Omohundro">Steve Omohundro</a></li>

<li><a href="Huw_Price" title="Huw Price">Huw Price</a></li>

<li><a href="Martin_Rees" title="Martin Rees">Martin Rees</a></li>

<li><a href="Stuart_J._Russell" title="Stuart J. Russell">Stuart J. Russell</a></li>

<li><a href="Ilya_Sutskever" title="Ilya Sutskever">Ilya Sutskever</a></li>

<li><a href="Jaan_Tallinn" title="Jaan Tallinn">Jaan Tallinn</a></li>

<li><a href="Max_Tegmark" title="Max Tegmark">Max Tegmark</a></li>

<li><a href="Frank_Wilczek" title="Frank Wilczek">Frank Wilczek</a></li>

<li><a href="Roman_Yampolskiy" title="Roman Yampolskiy">Roman Yampolskiy</a></li>

<li><a href="Eliezer_Yudkowsky" title="Eliezer Yudkowsky">Eliezer Yudkowsky</a></li></ul>

</div></td></tr><tr><th scope="row" class="navbox-group" style="width:1%">Other</th><td class="navbox-list-with-group navbox-list navbox-even hlist" style="width:100%;padding:0"><div style="padding:0 0.25em">

<ul><li><a href="Artificial_Intelligence_Act" title="Artificial Intelligence Act">Artificial Intelligence Act</a></li>

<li><a href="Do_You_Trust_This_Computer%3F" title="Do You Trust This Computer?">Do You Trust This Computer?</a></li>

<li><a href="Human_Compatible" title="Human Compatible">Human Compatible</a></li>

<li><a href="Open_letter_on_artificial_intelligence_(2015)" class="mw-redirect" title="Open letter on artificial intelligence (2015)">Open letter on artificial intelligence (2015)</a></li>

<li><a href="Our_Final_Invention" title="Our Final Invention">Our Final Invention</a></li>

<li><a href="Roko's_basilisk" title="Roko's basilisk">Roko's basilisk</a></li>

<li><a href="Statement_on_AI_risk_of_extinction" class="mw-redirect" title="Statement on AI risk of extinction">Statement on AI risk of extinction</a></li>

<li><a href="Superintelligence%3A_Paths%2C_Dangers%2C_Strategies" title="Superintelligence: Paths, Dangers, Strategies">Superintelligence: Paths, Dangers, Strategies</a></li>

<li><a href="The_Precipice%3A_Existential_Risk_and_the_Future_of_Humanity" title="The Precipice: Existential Risk and the Future of Humanity">The Precipice</a></li></ul>

</div></td></tr><tr><td class="navbox-abovebelow" colspan="2"><div> Category</div></td></tr></tbody></table></div></div><div><div class="zim-footer">

This article is issued from <a class="external text" title="Last edited on 2025-07-21" href="https://en.wikipedia.org/wiki/?title=AI_alignment&oldid=1301770021">Wikipedia</a>. The text is available under <a class="external text" href="https://creativecommons.org/licenses/by-sa/4.0/deed.en">Creative Commons Attribution-Share Alike 4.0</a> unless otherwise noted. Additional terms may apply for the media files.

</div>

</div></div>

</div>

</main>

</div>

</body></html>